@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/src/replication/WalStreamManager.ts

@@ -1,213 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { hrtime } from 'node:process';
-
-import * as storage from '../storage/storage-index.js';
-import * as util from '../util/util-index.js';
-
-import { DefaultErrorRateLimiter } from './ErrorRateLimiter.js';
-import { WalStreamRunner } from './WalStreamRunner.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
-import { container, logger } from '@powersync/lib-services-framework';
-
-// 5 minutes
-const PING_INTERVAL = 1_000_000_000n * 300n;
-
-export class WalStreamManager {
-  private streams = new Map<number, WalStreamRunner>();
-
-  private system: CorePowerSyncSystem;
-
-  private stopped = false;
-
-  // First ping is only after 5 minutes, not when starting
-  private lastPing = hrtime.bigint();
-
-  private storage: storage.BucketStorageFactory;
-
-  /**
-   * This limits the effect of retries when there is a persistent issue.
-   */
-  private rateLimiter = new DefaultErrorRateLimiter();
-
-  constructor(system: CorePowerSyncSystem) {
-    this.system = system;
-    this.storage = system.storage;
-  }
-
-  start() {
-    this.runLoop().catch((e) => {
-      logger.error(`Fatal WalStream error`, e);
-      container.reporter.captureException(e);
-      setTimeout(() => {
-        process.exit(1);
-      }, 1000);
-    });
-  }
-
-  async stop() {
-    this.stopped = true;
-    let promises: Promise<void>[] = [];
-    for (let stream of this.streams.values()) {
-      promises.push(stream.stop());
-    }
-    await Promise.all(promises);
-  }
-
-  private async runLoop() {
-    const configured_sync_rules = await util.loadSyncRules(this.system.config);
-    let configured_lock: storage.ReplicationLock | undefined = undefined;
-    if (configured_sync_rules != null) {
-      logger.info('Loading sync rules from configuration');
-      try {
-        // Configure new sync rules, if it has changed.
-        // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
-        const { updated, persisted_sync_rules, lock } = await this.storage.configureSyncRules(configured_sync_rules!, {
-          lock: true
-        });
-        if (lock) {
-          configured_lock = lock;
-        }
-      } catch (e) {
-        // Log, but continue with previous sync rules
-        logger.error(`Failed to load sync rules from configuration`, e);
-      }
-    } else {
-      logger.info('No sync rules configured - configure via API');
-    }
-    while (!this.stopped) {
-      await container.probes.touch();
-      try {
-        const pool = this.system.pgwire_pool;
-        if (pool) {
-          await this.refresh({ configured_lock });
-          // The lock is only valid on the first refresh.
-          configured_lock = undefined;
-
-          // TODO: Ping on all connections when we have multiple
-          // Perhaps WalStreamRunner would be a better place to do pings?
-          // We don't ping while in error retry back-off, to avoid having too
-          // many authentication failures.
-          if (this.rateLimiter.mayPing()) {
-            await this.ping(pool);
-          }
-        }
-      } catch (e) {
-        logger.error(`Failed to refresh wal streams`, e);
-      }
-      await new Promise((resolve) => setTimeout(resolve, 5000));
-    }
-  }
-
-  /**
-   * Postgres on RDS writes performs a WAL checkpoint every 5 minutes by default, which creates a new 64MB file.
-   *
-   * The old WAL files are only deleted once no replication slot still references it.
-   *
-   * Unfortunately, when there are no changes to the db, the database creates new WAL files without the replication slot
-   * advancing**.
-   *
-   * As a workaround, we write a new message every couple of minutes, to make sure that the replication slot advances.
-   *
-   * **This may be a bug in pgwire or how we're using it.
-   */
-  private async ping(db: pgwire.PgClient) {
-    const now = hrtime.bigint();
-    if (now - this.lastPing >= PING_INTERVAL) {
-      try {
-        await db.query(`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`);
-      } catch (e) {
-        logger.warn(`Failed to ping`, e);
-      }
-      this.lastPing = now;
-    }
-  }
-
-  private async refresh(options?: { configured_lock?: storage.ReplicationLock }) {
-    if (this.stopped) {
-      return;
-    }
-
-    let configured_lock = options?.configured_lock;
-
-    const existingStreams = new Map<number, WalStreamRunner>(this.streams.entries());
-    const replicating = await this.storage.getReplicatingSyncRules();
-    const newStreams = new Map<number, WalStreamRunner>();
-    for (let syncRules of replicating) {
-      const existing = existingStreams.get(syncRules.id);
-      if (existing && !existing.stopped) {
-        // No change
-        existingStreams.delete(syncRules.id);
-        newStreams.set(syncRules.id, existing);
-      } else if (existing && existing.stopped) {
-        // Stopped (e.g. fatal error, slot rename).
-        // Remove from the list. Next refresh call will restart the stream.
-        existingStreams.delete(syncRules.id);
-      } else {
-        // New (or resume after restart)
-        try {
-          let lock: storage.ReplicationLock;
-          if (configured_lock?.sync_rules_id == syncRules.id) {
-            lock = configured_lock;
-          } else {
-            lock = await syncRules.lock();
-          }
-          const parsed = syncRules.parsed();
-          const storage = this.storage.getInstance(parsed);
-          const stream = new WalStreamRunner({
-            factory: this.storage,
-            storage: storage,
-            source_db: this.system.config.connection!,
-            lock,
-            rateLimiter: this.rateLimiter
-          });
-          newStreams.set(syncRules.id, stream);
-          stream.start();
-        } catch (e) {
-          // Could be a sync rules parse error,
-          // for example from stricter validation that was added.
-          // This will be retried every couple of seconds.
-          // When new (valid) sync rules are deployed and processed, this one be disabled.
-          logger.error(`Failed to start replication for ${syncRules.slot_name}`, e);
-        }
-      }
-    }
-
-    this.streams = newStreams;
-
-    // TODO: Should this termination be happening in the "background" instead?
-    // That becomes tricky to manage
-
-    for (let stream of existingStreams.values()) {
-      // Old - stop and remove.
-      try {
-        await stream.terminate();
-      } catch (e) {
-        // This will be retried
-        logger.warn(`Failed to terminate ${stream.slot_name}`, e);
-      }
-    }
-
-    // Sync rules stopped previously or by a different process.
-    const stopped = await this.storage.getStoppedSyncRules();
-    for (let syncRules of stopped) {
-      try {
-        const lock = await syncRules.lock();
-        try {
-          const parsed = syncRules.parsed();
-          const storage = this.storage.getInstance(parsed);
-          const stream = new WalStreamRunner({
-            factory: this.storage,
-            storage: storage,
-            source_db: this.system.config.connection!,
-            lock
-          });
-          await stream.terminate();
-        } finally {
-          await lock.release();
-        }
-      } catch (e) {
-        logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
-      }
-    }
-  }
-}

package/src/replication/WalStreamRunner.ts

@@ -1,180 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-
-import * as storage from '../storage/storage-index.js';
-import * as util from '../util/util-index.js';
-
-import { ErrorRateLimiter } from './ErrorRateLimiter.js';
-import { MissingReplicationSlotError, WalStream } from './WalStream.js';
-import { ResolvedConnection } from '../util/config/types.js';
-import { container, logger } from '@powersync/lib-services-framework';
-
-export interface WalStreamRunnerOptions {
-  factory: storage.BucketStorageFactory;
-  storage: storage.SyncRulesBucketStorage;
-  source_db: ResolvedConnection;
-  lock: storage.ReplicationLock;
-  rateLimiter?: ErrorRateLimiter;
-}
-
-export class WalStreamRunner {
-  private abortController = new AbortController();
-
-  private runPromise?: Promise<void>;
-
-  private connections: util.PgManager | null = null;
-
-  private rateLimiter?: ErrorRateLimiter;
-
-  constructor(public options: WalStreamRunnerOptions) {
-    this.rateLimiter = options.rateLimiter;
-  }
-
-  start() {
-    this.runPromise = this.run();
-  }
-
-  get slot_name() {
-    return this.options.storage.slot_name;
-  }
-
-  get stopped() {
-    return this.abortController.signal.aborted;
-  }
-
-  async run() {
-    try {
-      await this.replicateLoop();
-    } catch (e) {
-      // Fatal exception
-      container.reporter.captureException(e, {
-        metadata: {
-          replication_slot: this.slot_name
-        }
-      });
-      logger.error(`Replication failed on ${this.slot_name}`, e);
-
-      if (e instanceof MissingReplicationSlotError) {
-        // This stops replication on this slot, and creates a new slot
-        await this.options.storage.factory.slotRemoved(this.slot_name);
-      }
-    } finally {
-      this.abortController.abort();
-    }
-    await this.options.lock.release();
-  }
-
-  async replicateLoop() {
-    while (!this.stopped) {
-      await this.replicateOnce();
-
-      if (!this.stopped) {
-        await new Promise((resolve) => setTimeout(resolve, 5000));
-      }
-    }
-  }
-
-  async replicateOnce() {
-    // New connections on every iteration (every error with retry),
-    // otherwise we risk repeating errors related to the connection,
-    // such as caused by cached PG schemas.
-    let connections = new util.PgManager(this.options.source_db, {
-      // Pool connections are only used intermittently.
-      idleTimeout: 30_000,
-      maxSize: 2
-    });
-    this.connections = connections;
-    try {
-      await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
-      if (this.stopped) {
-        return;
-      }
-      const stream = new WalStream({
-        abort_signal: this.abortController.signal,
-        factory: this.options.factory,
-        storage: this.options.storage,
-        connections
-      });
-      await stream.replicate();
-    } catch (e) {
-      logger.error(`Replication error`, e);
-      if (e.cause != null) {
-        // Example:
-        // PgError.conn_ended: Unable to do postgres query on ended connection
-        //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
-        //     at stream.next (<anonymous>)
-        //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
-        //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
-        //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
-        //     ...
-        //   cause: TypeError: match is not iterable
-        //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
-        //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
-        //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
-        //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
-        //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
-        //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
-        //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
-        //   [Symbol(pg.ErrorCode)]: 'conn_ended',
-        //   [Symbol(pg.ErrorResponse)]: undefined
-        // }
-        // Without this additional log, the cause would not be visible in the logs.
-        logger.error(`cause`, e.cause);
-      }
-      if (e instanceof MissingReplicationSlotError) {
-        throw e;
-      } else {
-        // Report the error if relevant, before retrying
-        container.reporter.captureException(e, {
-          metadata: {
-            replication_slot: this.slot_name
-          }
-        });
-        // This sets the retry delay
-        this.rateLimiter?.reportError(e);
-      }
-    } finally {
-      this.connections = null;
-      if (connections != null) {
-        await connections.end();
-      }
-    }
-  }
-
-  /**
-   * This will also release the lock if start() was called earlier.
-   */
-  async stop(options?: { force?: boolean }) {
-    logger.info(`${this.slot_name} Stopping replication`);
-    // End gracefully
-    this.abortController.abort();
-
-    if (options?.force) {
-      // destroy() is more forceful.
-      await this.connections?.destroy();
-    }
-    await this.runPromise;
-  }
-
-  /**
-   * Terminate this replication stream. This drops the replication slot and deletes the replication data.
-   *
-   * Stops replication if needed.
-   */
-  async terminate(options?: { force?: boolean }) {
-    logger.info(`${this.slot_name} Terminating replication`);
-    await this.stop(options);
-
-    const slotName = this.slot_name;
-    const db = await pgwire.connectPgWire(this.options.source_db, { type: 'standard' });
-    try {
-      await db.query({
-        statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
-        params: [{ type: 'varchar', value: slotName }]
-      });
-    } finally {
-      await db.end();
-    }
-
-    await this.options.storage.terminate();
-  }
-}

package/src/replication/util.ts

@@ -1,76 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-
-import * as util from '../util/util-index.js';
-import { ReplicationColumn, ReplicationIdentity } from './PgRelation.js';
-
-export interface ReplicaIdentityResult {
-  columns: ReplicationColumn[];
-  replicationIdentity: ReplicationIdentity;
-}
-
-export async function getPrimaryKeyColumns(
-  db: pgwire.PgClient,
-  relationId: number,
-  mode: 'primary' | 'replident'
-): Promise<ReplicationColumn[]> {
-  const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`;
-  const attrRows = await util.retriedQuery(db, {
-    statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
-                                    FROM pg_index i
-                                             JOIN pg_attribute a
-                                                  ON a.attrelid = i.indrelid
-                                                      AND a.attnum = ANY (i.indkey)
-                                    WHERE i.indrelid = $1::oid
-                                      AND ${indexFlag}
-                                      AND a.attnum > 0
-                                      ORDER BY a.attnum`,
-    params: [{ value: relationId, type: 'int4' }]
-  });
-
-  return attrRows.rows.map((row) => {
-    return { name: row[0] as string, typeOid: row[1] as number };
-  });
-}
-
-export async function getAllColumns(db: pgwire.PgClient, relationId: number): Promise<ReplicationColumn[]> {
-  const attrRows = await util.retriedQuery(db, {
-    statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
-                                    FROM pg_attribute a
-                                    WHERE a.attrelid = $1::oid
-                                      AND attnum > 0
-                                    ORDER BY a.attnum`,
-    params: [{ type: 'varchar', value: relationId }]
-  });
-  return attrRows.rows.map((row) => {
-    return { name: row[0] as string, typeOid: row[1] as number };
-  });
-}
-
-export async function getReplicationIdentityColumns(
-  db: pgwire.PgClient,
-  relationId: number
-): Promise<ReplicaIdentityResult> {
-  const rows = await util.retriedQuery(db, {
-    statement: `SELECT CASE relreplident
-        WHEN 'd' THEN 'default'
-        WHEN 'n' THEN 'nothing'
-        WHEN 'f' THEN 'full'
-        WHEN 'i' THEN 'index'
-     END AS replica_identity
-FROM pg_class
-WHERE oid = $1::oid LIMIT 1`,
-    params: [{ type: 'int8', value: relationId }]
-  });
-  const idType: string = rows.rows[0]?.[0];
-  if (idType == 'nothing' || idType == null) {
-    return { replicationIdentity: 'nothing', columns: [] };
-  } else if (idType == 'full') {
-    return { replicationIdentity: 'full', columns: await getAllColumns(db, relationId) };
-  } else if (idType == 'default') {
-    return { replicationIdentity: 'default', columns: await getPrimaryKeyColumns(db, relationId, 'primary') };
-  } else if (idType == 'index') {
-    return { replicationIdentity: 'index', columns: await getPrimaryKeyColumns(db, relationId, 'replident') };
-  } else {
-    return { replicationIdentity: 'nothing', columns: [] };
-  }
-}

package/src/routes/endpoints/dev.ts

@@ -1,199 +0,0 @@
-import * as t from 'ts-codec';
-import * as pgwire from '@powersync/service-jpgwire';
-import { errors, router, schema } from '@powersync/lib-services-framework';
-
-import * as util from '../../util/util-index.js';
-import { authDevUser, authUser, endpoint, issueDevToken, issueLegacyDevToken, issuePowerSyncToken } from '../auth.js';
-import { routeDefinition } from '../router.js';
-
-const AuthParams = t.object({
-  user: t.string,
-  password: t.string
-});
-
-// For legacy web client only. Remove soon.
-export const auth = routeDefinition({
-  path: '/auth.json',
-  method: router.HTTPMethod.POST,
-  validator: schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
-  handler: async (payload) => {
-    const { user, password } = payload.params;
-    const config = payload.context.system.config;
-
-    if (config.dev.demo_auth == false || config.dev.demo_password == null) {
-      throw new errors.AuthorizationError(['Demo auth disabled']);
-    }
-
-    if (password == config.dev.demo_password) {
-      const token = await issueLegacyDevToken(payload.request, user, payload.context.system.config);
-      return { token, user_id: user, endpoint: endpoint(payload.request) };
-    } else {
-      throw new errors.AuthorizationError(['Authentication failed']);
-    }
-  }
-});
-
-export const auth2 = routeDefinition({
-  path: '/dev/auth.json',
-  method: router.HTTPMethod.POST,
-  validator: schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
-  handler: async (payload) => {
-    const { user, password } = payload.params;
-    const config = payload.context.system.config;
-
-    if (config.dev.demo_auth == false || config.dev.demo_password == null) {
-      throw new errors.AuthorizationError(['Demo auth disabled']);
-    }
-
-    if (password == config.dev.demo_password) {
-      const token = await issueDevToken(payload.request, user, payload.context.system.config);
-      return { token, user_id: user };
-    } else {
-      throw new errors.AuthorizationError(['Authentication failed']);
-    }
-  }
-});
-
-const TokenParams = t.object({});
-
-export const token = routeDefinition({
-  path: '/dev/token.json',
-  method: router.HTTPMethod.POST,
-  validator: schema.createTsCodecValidator(TokenParams, { allowAdditional: true }),
-  authorize: authDevUser,
-  handler: async (payload) => {
-    const { user_id } = payload.context;
-    const outToken = await issuePowerSyncToken(payload.request, user_id!, payload.context.system.config);
-    return { token: outToken, user_id: user_id, endpoint: endpoint(payload.request) };
-  }
-});
-
-const OpType = {
-  PUT: 'PUT',
-  PATCH: 'PATCH',
-  DELETE: 'DELETE'
-};
-
-const CrudEntry = t.object({
-  op: t.Enum(OpType),
-  type: t.string,
-  id: t.string,
-  op_id: t.number.optional(),
-  data: t.any.optional()
-});
-
-const CrudRequest = t.object({
-  data: t.array(CrudEntry),
-  write_checkpoint: t.boolean.optional()
-});
-
-export const crud = routeDefinition({
-  path: '/crud.json',
-  method: router.HTTPMethod.POST,
-  validator: schema.createTsCodecValidator(CrudRequest, { allowAdditional: true }),
-  authorize: authUser,
-
-  handler: async (payload) => {
-    const { user_id, system } = payload.context;
-
-    const pool = system.requirePgPool();
-
-    if (!system.config.dev.crud_api) {
-      throw new Error('CRUD api disabled');
-    }
-
-    const params = payload.params;
-
-    let statements: pgwire.Statement[] = [];
-
-    // Implementation note:
-    // Postgres does automatic "assigment cast" for query literals,
-    // e.g. a string literal to uuid. However, the same doesn't apply
-    // to query parameters.
-    // To handle those automatically, we use `json_populate_record`
-    // to automatically cast to the correct types.
-
-    for (let op of params.data) {
-      const table = util.escapeIdentifier(op.type);
-      if (op.op == 'PUT') {
-        const data = op.data as Record<string, any>;
-        const with_id = { ...data, id: op.id };
-
-        const columnsEscaped = Object.keys(with_id).map(util.escapeIdentifier);
-        const columnsJoined = columnsEscaped.join(', ');
-
-        let updateClauses: string[] = [];
-
-        for (let key of Object.keys(data)) {
-          updateClauses.push(`${util.escapeIdentifier(key)} = EXCLUDED.${util.escapeIdentifier(key)}`);
-        }
-
-        const updateClause = updateClauses.length > 0 ? `DO UPDATE SET ${updateClauses.join(', ')}` : `DO NOTHING`;
-
-        const statement = `
-          WITH data_row AS (
-              SELECT (json_populate_record(null::${table}, $1::json)).*
-          )
-          INSERT INTO ${table} (${columnsJoined})
-          SELECT ${columnsJoined} FROM data_row
-          ON CONFLICT(id) ${updateClause}`;
-
-        statements.push({
-          statement: statement,
-          params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
-        });
-      } else if (op.op == 'PATCH') {
-        const data = op.data as Record<string, any>;
-        const with_id = { ...data, id: op.id };
-
-        let updateClauses: string[] = [];
-
-        for (let key of Object.keys(data)) {
-          updateClauses.push(`${util.escapeIdentifier(key)} = data_row.${util.escapeIdentifier(key)}`);
-        }
-
-        const statement = `
-          WITH data_row AS (
-              SELECT (json_populate_record(null::${table}, $1::json)).*
-          )
-          UPDATE ${table}
-          SET ${updateClauses.join(', ')}
-          FROM data_row
-          WHERE ${table}.id = data_row.id`;
-
-        statements.push({
-          statement: statement,
-          params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
-        });
-      } else if (op.op == 'DELETE') {
-        statements.push({
-          statement: `
-          WITH data_row AS (
-            SELECT (json_populate_record(null::${table}, $1::json)).*
-          )
-          DELETE FROM ${table}
-          USING data_row
-          WHERE ${table}.id = data_row.id`,
-          params: [{ type: 'varchar', value: JSON.stringify({ id: op.id }) }]
-        });
-      }
-    }
-    await pool.query(...statements);
-
-    const storage = system.storage;
-    if (payload.params.write_checkpoint === true) {
-      const write_checkpoint = await util.createWriteCheckpoint(pool, storage, payload.context.user_id!);
-      return { write_checkpoint: String(write_checkpoint) };
-    } else if (payload.params.write_checkpoint === false) {
-      return {};
-    } else {
-      // Legacy
-      const checkpoint = await util.getClientCheckpoint(pool, storage);
-      return {
-        checkpoint
-      };
-    }
-  }
-});
-
-export const DEV_ROUTES = [auth, auth2, token, crud];