@powersync/service-core 0.0.0-dev-20240620165206

package/src/migrations/executor.ts

@@ -0,0 +1,87 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as defs from './definitions.js';
+import { MigrationStore } from './store/migration-store.js';
+
+type ExecuteParams = {
+  migrations: defs.Migration[];
+  state?: defs.MigrationState;
+
+  direction: defs.Direction;
+  count?: number;
+};
+
+export async function* execute(params: ExecuteParams): AsyncGenerator<defs.ExecutedMigration> {
+  let migrations = [...params.migrations];
+  if (params.direction === defs.Direction.Down) {
+    migrations = migrations.reverse();
+  }
+
+  let index = 0;
+
+  if (params.state) {
+    // Find the index of the last run
+    index = migrations.findIndex((migration) => {
+      return migration.name === params.state!.last_run;
+    });
+
+    if (index === -1) {
+      throw new Error(`The last run migration ${params.state?.last_run} was not found in the given set of migrations`);
+    }
+
+    // If we are migrating down then we want to include the last run migration, otherwise we want to start at the next one
+    if (params.direction === defs.Direction.Up) {
+      index += 1;
+    }
+  }
+
+  migrations = migrations.slice(index);
+
+  let i = 0;
+  for (const migration of migrations) {
+    if (params.count && params.count === i) {
+      return;
+    }
+
+    logger.info(`Executing ${migration.name} (${params.direction})`);
+    try {
+      switch (params.direction) {
+        case defs.Direction.Up: {
+          await migration.up();
+          break;
+        }
+        case defs.Direction.Down: {
+          await migration.down();
+          break;
+        }
+      }
+      logger.debug(`Success`);
+    } catch (err) {
+      logger.error(`Failed`, err);
+      process.exit(1);
+    }
+
+    yield {
+      name: migration.name,
+      direction: params.direction,
+      timestamp: new Date()
+    };
+
+    i++;
+  }
+}
+
+type WriteLogsParams = {
+  store: MigrationStore;
+  state?: defs.MigrationState;
+  log_stream: Iterable<defs.ExecutedMigration> | AsyncIterable<defs.ExecutedMigration>;
+};
+export const writeLogsToStore = async (params: WriteLogsParams): Promise<void> => {
+  const log = [...(params.state?.log || [])];
+  for await (const migration of params.log_stream) {
+    log.push(migration);
+    await params.store.save({
+      last_run: migration.name,
+      log: log
+    });
+  }
+};

package/src/migrations/migrations-index.ts

@@ -0,0 +1,3 @@
+export * from './definitions.js';
+export * from './executor.js';
+export * from './migrations.js';

package/src/migrations/migrations.ts

@@ -0,0 +1,118 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+import * as db from '../db/db-index.js';
+import * as util from '../util/util-index.js';
+import * as locks from '../locks/locks-index.js';
+import { Direction } from './definitions.js';
+import { createMongoMigrationStore } from './store/migration-store.js';
+import { execute, writeLogsToStore } from './executor.js';
+
+const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
+const MONGO_LOCK_PROCESS = 'migrations';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const MIGRATIONS_DIR = path.join(__dirname, '/db/migrations');
+
+export type MigrationOptions = {
+  direction: Direction;
+  runner_config: util.RunnerConfig;
+};
+
+/**
+ * Loads migrations and injects a custom context for loading the specified
+ * runner configuration.
+ */
+const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) => {
+  const files = await fs.readdir(dir);
+  const migrations = files.filter((file) => {
+    return path.extname(file) === '.js';
+  });
+
+  const context: util.MigrationContext = {
+    runner_config
+  };
+
+  return await Promise.all(
+    migrations.map(async (migration) => {
+      const module = await import(path.resolve(dir, migration));
+      return {
+        name: path.basename(migration).replace(path.extname(migration), ''),
+        up: () => module.up(context),
+        down: () => module.down(context)
+      };
+    })
+  );
+};
+
+/**
+ * Runs migration scripts exclusively using Mongo locks
+ */
+export const migrate = async (options: MigrationOptions) => {
+  const { direction, runner_config } = options;
+
+  /**
+   * Try and get Mongo from config file.
+   * But this might not be available in Journey Micro as we use the standard Mongo.
+   */
+
+  const config = await util.loadConfig(runner_config);
+  const { storage } = config;
+
+  const client = db.mongo.createMongoClient(storage);
+  await client.connect();
+
+  const clientDB = client.db(storage.database);
+  const collection = clientDB.collection<locks.Lock>(DEFAULT_MONGO_LOCK_COLLECTION);
+
+  const manager = locks.createMongoLockManager(collection, {
+    name: MONGO_LOCK_PROCESS
+  });
+
+  // Only one process should execute this at a time.
+  const lockId = await manager.acquire();
+
+  if (!lockId) {
+    throw new Error('Could not acquire lock_id');
+  }
+
+  let isReleased = false;
+  const releaseLock = async () => {
+    if (isReleased) {
+      return;
+    }
+    await manager.release(lockId);
+    isReleased = true;
+  };
+
+  // For the case where the migration is terminated
+  process.addListener('beforeExit', releaseLock);
+
+  try {
+    const migrations = await loadMigrations(MIGRATIONS_DIR, runner_config);
+
+    // Use the provided config to connect to Mongo
+    const store = createMongoMigrationStore(client);
+
+    const state = await store.load();
+
+    const logStream = execute({
+      direction: direction,
+      migrations,
+      state
+    });
+
+    await writeLogsToStore({
+      log_stream: logStream,
+      store,
+      state
+    });
+  } finally {
+    await releaseLock();
+    await client.close(true);
+    process.removeListener('beforeExit', releaseLock);
+  }
+};

package/src/migrations/store/migration-store.ts

@@ -0,0 +1,63 @@
+import Mongo from 'mongodb';
+import * as path from 'path';
+import * as defs from '../definitions.js';
+
+export type MigrationStore = {
+  load: () => Promise<defs.MigrationState | undefined>;
+  save: (state: defs.MigrationState) => Promise<void>;
+};
+
+/**
+ * A custom store for node-migrate which is used to save and load migrations that have
+ * been operated on to mongo.
+ */
+export const createMongoMigrationStore = (client: Mongo.MongoClient): MigrationStore => {
+  const collection = client.db().collection<defs.MigrationState>('migrations');
+
+  return {
+    load: async () => {
+      const state_entry = await collection.findOne();
+      if (!state_entry) {
+        return;
+      }
+
+      const { _id, ...state } = state_entry;
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to save
+       * state as `lastRun`.
+       */
+      let last_run = state.last_run;
+      if ('lastRun' in state) {
+        last_run = (state as any).lastRun;
+      }
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to include the
+       * file extension in migration names. This strips that extension off if it exists
+       */
+      const extension = path.extname(last_run);
+      if (extension) {
+        last_run = last_run.replace(extension, '');
+      }
+
+      return {
+        last_run,
+        log: state.log || []
+      };
+    },
+
+    save: async (state: defs.MigrationState) => {
+      await collection.replaceOne(
+        {},
+        {
+          last_run: state.last_run,
+          log: state.log
+        },
+        {
+          upsert: true
+        }
+      );
+    }
+  };
+};

package/src/replication/ErrorRateLimiter.ts

@@ -0,0 +1,50 @@
+import { setTimeout } from 'timers/promises';
+
+export interface ErrorRateLimiter {
+  waitUntilAllowed(options?: { signal?: AbortSignal }): Promise<void>;
+  reportError(e: any): void;
+
+  mayPing(): boolean;
+}
+
+export class DefaultErrorRateLimiter implements ErrorRateLimiter {
+  nextAllowed: number = Date.now();
+
+  async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
+    const delay = Math.max(0, this.nextAllowed - Date.now());
+    // Minimum delay between connections, even without errors
+    this.setDelay(500);
+    await setTimeout(delay, undefined, { signal: options?.signal });
+  }
+
+  mayPing(): boolean {
+    return Date.now() >= this.nextAllowed;
+  }
+
+  reportError(e: any): void {
+    const message = (e.message as string) ?? '';
+    if (message.includes('password authentication failed')) {
+      // Wait 15 minutes, to avoid triggering Supabase's fail2ban
+      this.setDelay(900_000);
+    } else if (message.includes('ENOTFOUND')) {
+      // DNS lookup issue - incorrect URI or deleted instance
+      this.setDelay(120_000);
+    } else if (message.includes('ECONNREFUSED')) {
+      // Could be fail2ban or similar
+      this.setDelay(120_000);
+    } else if (
+      message.includes('Unable to do postgres query on ended pool') ||
+      message.includes('Postgres unexpectedly closed connection')
+    ) {
+      // Connection timed out - ignore / immediately retry
+      // We don't explicitly set the delay to 0, since there could have been another error that
+      // we need to respect.
+    } else {
+      this.setDelay(30_000);
+    }
+  }
+
+  private setDelay(delay: number) {
+    this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
+  }
+}

package/src/replication/PgRelation.ts

@@ -0,0 +1,42 @@
+import { PgoutputRelation } from '@powersync/service-jpgwire';
+
+export interface PgRelation {
+  readonly relationId: number;
+  readonly schema: string;
+  readonly name: string;
+  readonly replicaIdentity: ReplicationIdentity;
+  readonly replicationColumns: ReplicationColumn[];
+}
+
+export type ReplicationIdentity = 'default' | 'nothing' | 'full' | 'index';
+
+export interface ReplicationColumn {
+  readonly name: string;
+  readonly typeOid: number;
+}
+
+export function getReplicaIdColumns(relation: PgoutputRelation): ReplicationColumn[] {
+  if (relation.replicaIdentity == 'nothing') {
+    return [];
+  } else {
+    return relation.columns.filter((c) => (c.flags & 0b1) != 0).map((c) => ({ name: c.name, typeOid: c.typeOid }));
+  }
+}
+export function getRelId(source: PgoutputRelation): number {
+  // Source types are wrong here
+  const relId = (source as any).relationOid as number;
+  if (relId == null || typeof relId != 'number') {
+    throw new Error(`No relation id!`);
+  }
+  return relId;
+}
+
+export function getPgOutputRelation(source: PgoutputRelation): PgRelation {
+  return {
+    name: source.name,
+    schema: source.schema,
+    relationId: getRelId(source),
+    replicaIdentity: source.replicaIdentity,
+    replicationColumns: getReplicaIdColumns(source)
+  };
+}

package/src/replication/WalConnection.ts

@@ -0,0 +1,227 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import { pgwireRows } from '@powersync/service-jpgwire';
+import { DEFAULT_TAG, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import { ReplicationError, TableInfo } from '@powersync/service-types';
+
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
+
+import { ReplicaIdentityResult, getReplicationIdentityColumns } from './util.js';
+/**
+ * Connection that _manages_ WAL, but does not do streaming.
+ */
+export class WalConnection {
+  db: pgwire.PgClient;
+  connectionTag = DEFAULT_TAG;
+  publication_name = 'powersync';
+
+  sync_rules: SqlSyncRules;
+
+  /**
+   * db can be a PgConnection or PgPool.
+   *
+   * No transactions are used here, but it is up to the client to ensure
+   * nothing here is called in the middle of another transaction if
+   * PgConnection is used.
+   */
+  constructor(options: { db: pgwire.PgClient; sync_rules: SqlSyncRules }) {
+    this.db = options.db;
+    this.sync_rules = options.sync_rules;
+  }
+
+  async checkSourceConfiguration() {
+    await checkSourceConfiguration(this.db);
+  }
+
+  async getDebugTableInfo(tablePattern: TablePattern, name: string, relationId: number | null): Promise<TableInfo> {
+    const schema = tablePattern.schema;
+    let id_columns_result: ReplicaIdentityResult | undefined = undefined;
+    let id_columns_error = null;
+
+    if (relationId != null) {
+      try {
+        id_columns_result = await getReplicationIdentityColumns(this.db, relationId);
+      } catch (e) {
+        id_columns_error = { level: 'fatal', message: e.message };
+      }
+    }
+
+    const id_columns = id_columns_result?.columns ?? [];
+
+    const sourceTable = new storage.SourceTable(0, this.connectionTag, relationId ?? 0, schema, name, id_columns, true);
+
+    const syncData = this.sync_rules.tableSyncsData(sourceTable);
+    const syncParameters = this.sync_rules.tableSyncsParameters(sourceTable);
+
+    if (relationId == null) {
+      return {
+        schema: schema,
+        name: name,
+        pattern: tablePattern.isWildcard ? tablePattern.tablePattern : undefined,
+        replication_id: [],
+        data_queries: syncData,
+        parameter_queries: syncParameters,
+        // Also
+        errors: [{ level: 'warning', message: `Table ${sourceTable.qualifiedName} not found.` }]
+      };
+    }
+    if (id_columns.length == 0 && id_columns_error == null) {
+      let message = `No replication id found for ${sourceTable.qualifiedName}. Replica identity: ${id_columns_result?.replicationIdentity}.`;
+      if (id_columns_result?.replicationIdentity == 'default') {
+        message += ' Configure a primary key on the table.';
+      }
+      id_columns_error = { level: 'fatal', message };
+    }
+
+    let selectError = null;
+    try {
+      await util.retriedQuery(this.db, `SELECT * FROM ${sourceTable.escapedIdentifier} LIMIT 1`);
+    } catch (e) {
+      selectError = { level: 'fatal', message: e.message };
+    }
+
+    let replicateError = null;
+
+    const publications = await util.retriedQuery(this.db, {
+      statement: `SELECT tablename FROM pg_publication_tables WHERE pubname = $1 AND schemaname = $2 AND tablename = $3`,
+      params: [
+        { type: 'varchar', value: this.publication_name },
+        { type: 'varchar', value: tablePattern.schema },
+        { type: 'varchar', value: name }
+      ]
+    });
+    if (publications.rows.length == 0) {
+      replicateError = {
+        level: 'fatal',
+        message: `Table ${sourceTable.qualifiedName} is not part of publication '${this.publication_name}'. Run: \`ALTER PUBLICATION ${this.publication_name} ADD TABLE ${sourceTable.qualifiedName}\`.`
+      };
+    }
+
+    return {
+      schema: schema,
+      name: name,
+      pattern: tablePattern.isWildcard ? tablePattern.tablePattern : undefined,
+      replication_id: id_columns.map((c) => c.name),
+      data_queries: syncData,
+      parameter_queries: syncParameters,
+      errors: [id_columns_error, selectError, replicateError].filter((error) => error != null) as ReplicationError[]
+    };
+  }
+
+  async getDebugTablesInfo(tablePatterns: TablePattern[]): Promise<PatternResult[]> {
+    let result: PatternResult[] = [];
+
+    for (let tablePattern of tablePatterns) {
+      const schema = tablePattern.schema;
+
+      let patternResult: PatternResult = {
+        schema: schema,
+        pattern: tablePattern.tablePattern,
+        wildcard: tablePattern.isWildcard
+      };
+      result.push(patternResult);
+
+      if (tablePattern.isWildcard) {
+        patternResult.tables = [];
+        const prefix = tablePattern.tablePrefix;
+        const results = await util.retriedQuery(this.db, {
+          statement: `SELECT c.oid AS relid, c.relname AS table_name
+        FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = $1
+        AND c.relkind = 'r'
+        AND c.relname LIKE $2`,
+          params: [
+            { type: 'varchar', value: schema },
+            { type: 'varchar', value: tablePattern.tablePattern }
+          ]
+        });
+
+        for (let row of pgwireRows(results)) {
+          const name = row.table_name as string;
+          const relationId = row.relid as number;
+          if (!name.startsWith(prefix)) {
+            continue;
+          }
+          const details = await this.getDebugTableInfo(tablePattern, name, relationId);
+          patternResult.tables.push(details);
+        }
+      } else {
+        const results = await util.retriedQuery(this.db, {
+          statement: `SELECT c.oid AS relid, c.relname AS table_name
+        FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = $1
+        AND c.relkind = 'r'
+        AND c.relname = $2`,
+          params: [
+            { type: 'varchar', value: schema },
+            { type: 'varchar', value: tablePattern.tablePattern }
+          ]
+        });
+        if (results.rows.length == 0) {
+          // Table not found
+          const details = await this.getDebugTableInfo(tablePattern, tablePattern.name, null);
+          patternResult.table = details;
+        } else {
+          const row = pgwireRows(results)[0];
+          const name = row.table_name as string;
+          const relationId = row.relid as number;
+          patternResult.table = await this.getDebugTableInfo(tablePattern, name, relationId);
+        }
+      }
+    }
+    return result;
+  }
+}
+
+export interface PatternResult {
+  schema: string;
+  pattern: string;
+  wildcard: boolean;
+  tables?: TableInfo[];
+  table?: TableInfo;
+}
+
+export async function checkSourceConfiguration(db: pgwire.PgClient) {
+  // TODO: configurable
+  const publication_name = 'powersync';
+
+  // Check basic config
+  await util.retriedQuery(
+    db,
+    `DO $$
+BEGIN
+if current_setting('wal_level') is distinct from 'logical' then
+raise exception 'wal_level must be set to ''logical'', your database has it set to ''%''. Please edit your config file and restart PostgreSQL.', current_setting('wal_level');
+end if;
+if (current_setting('max_replication_slots')::int >= 1) is not true then
+raise exception 'Your max_replication_slots setting is too low, it must be greater than 1. Please edit your config file and restart PostgreSQL.';
+end if;
+if (current_setting('max_wal_senders')::int >= 1) is not true then
+raise exception 'Your max_wal_senders setting is too low, it must be greater than 1. Please edit your config file and restart PostgreSQL.';
+end if;
+end;
+$$ LANGUAGE plpgsql;`
+  );
+
+  // Check that publication exists
+  const rs = await util.retriedQuery(db, {
+    statement: `SELECT * FROM pg_publication WHERE pubname = $1`,
+    params: [{ type: 'varchar', value: publication_name }]
+  });
+  const row = pgwireRows(rs)[0];
+  if (row == null) {
+    throw new Error(
+      `Publication '${publication_name}' does not exist. Run: \`CREATE PUBLICATION ${publication_name} FOR ALL TABLES\`, or read the documentation for details.`
+    );
+  }
+  if (row.pubinsert == false || row.pubupdate == false || row.pubdelete == false || row.pubtruncate == false) {
+    throw new Error(
+      `Publication '${publication_name}' does not publish all changes. Create a publication using \`WITH (publish = "insert, update, delete, truncate")\` (the default).`
+    );
+  }
+  if (row.pubviaroot) {
+    throw new Error(`'${publication_name}' uses publish_via_partition_root, which is not supported.`);
+  }
+}