@powersync/service-module-postgres-storage 0.1.0

package/src/migrations/PostgresMigrationStore.ts

@@ -0,0 +1,70 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { migrations } from '@powersync/lib-services-framework';
+import { models } from '../types/types.js';
+import { sql } from '../utils/db.js';
+
+export type PostgresMigrationStoreOptions = {
+  db: lib_postgres.DatabaseClient;
+};
+
+export class PostgresMigrationStore implements migrations.MigrationStore {
+  constructor(protected options: PostgresMigrationStoreOptions) {}
+
+  protected get db() {
+    return this.options.db;
+  }
+
+  async init() {
+    await this.db.query(sql`
+      CREATE TABLE IF NOT EXISTS migrations (
+        id SERIAL PRIMARY KEY,
+        last_run TEXT,
+        LOG JSONB NOT NULL
+      );
+    `);
+  }
+
+  async clear() {
+    await this.db.query(sql`DELETE FROM migrations;`);
+  }
+
+  async load(): Promise<migrations.MigrationState | undefined> {
+    const res = await this.db.sql`
+      SELECT
+        last_run,
+        LOG
+      FROM
+        migrations
+      LIMIT
+        1
+    `
+      .decoded(models.Migration)
+      .first();
+
+    if (!res) {
+      return;
+    }
+
+    return {
+      last_run: res.last_run,
+      log: res.log
+    };
+  }
+
+  async save(state: migrations.MigrationState): Promise<void> {
+    await this.db.query(sql`
+      INSERT INTO
+        migrations (id, last_run, LOG)
+      VALUES
+        (
+          1,
+          ${{ type: 'varchar', value: state.last_run }},
+          ${{ type: 'jsonb', value: state.log }}
+        )
+      ON CONFLICT (id) DO UPDATE
+      SET
+        last_run = EXCLUDED.last_run,
+        LOG = EXCLUDED.log;
+    `);
+  }
+}

package/src/migrations/migration-utils.ts

@@ -0,0 +1,14 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { configFile } from '@powersync/service-types';
+import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js';
+import { STORAGE_SCHEMA_NAME } from '../utils/db.js';
+
+export const openMigrationDB = (config: configFile.BaseStorageConfig) => {
+  if (!isPostgresStorageConfig(config)) {
+    throw new Error(`Input storage configuration is not for Postgres`);
+  }
+  return new lib_postgres.DatabaseClient({
+    config: normalizePostgresStorageConfig(PostgresStorageConfig.decode(config)),
+    schema: STORAGE_SCHEMA_NAME
+  });
+};

package/src/migrations/scripts/1684951997326-init.ts

@@ -0,0 +1,141 @@
+import { migrations } from '@powersync/service-core';
+
+import { dropTables } from '../../utils/db.js';
+import { openMigrationDB } from '../migration-utils.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  /**
+   * Request an explicit connection which will automatically set the search
+   * path to the powersync schema
+   */
+  await client.transaction(async (db) => {
+    await db.sql`
+      CREATE SEQUENCE op_id_sequence AS int8 START
+      WITH
+        1
+    `.execute();
+
+    await db.sql`
+      CREATE SEQUENCE sync_rules_id_sequence AS int START
+      WITH
+        1
+    `.execute();
+
+    await db.sql`
+      CREATE TABLE bucket_data (
+        group_id integer NOT NULL,
+        bucket_name TEXT NOT NULL,
+        op_id bigint NOT NULL,
+        CONSTRAINT unique_id PRIMARY KEY (group_id, bucket_name, op_id),
+        op text NOT NULL,
+        source_table TEXT,
+        source_key bytea,
+        table_name TEXT,
+        row_id TEXT,
+        checksum bigint NOT NULL,
+        data TEXT,
+        target_op bigint
+      )
+    `.execute();
+
+    await db.sql`CREATE TABLE instance (id TEXT PRIMARY KEY) `.execute();
+
+    await db.sql`
+      CREATE TABLE sync_rules (
+        id INTEGER PRIMARY KEY,
+        state TEXT NOT NULL,
+        snapshot_done BOOLEAN NOT NULL DEFAULT FALSE,
+        last_checkpoint BIGINT,
+        last_checkpoint_lsn TEXT,
+        no_checkpoint_before TEXT,
+        slot_name TEXT,
+        last_checkpoint_ts TIMESTAMP WITH TIME ZONE,
+        last_keepalive_ts TIMESTAMP WITH TIME ZONE,
+        keepalive_op BIGINT,
+        last_fatal_error TEXT,
+        content TEXT NOT NULL
+      );
+    `.execute();
+
+    await db.sql`
+      CREATE TABLE bucket_parameters (
+        id BIGINT DEFAULT nextval('op_id_sequence') PRIMARY KEY,
+        group_id integer NOT NULL,
+        source_table TEXT NOT NULL,
+        source_key bytea NOT NULL,
+        lookup bytea NOT NULL,
+        --- Stored as text which is stringified with JSONBig
+        --- BigInts are not standard JSON, storing as JSONB seems risky
+        bucket_parameters text NOT NULL
+      );
+    `.execute();
+
+    await db.sql`
+      CREATE INDEX bucket_parameters_lookup_index ON bucket_parameters (group_id ASC, lookup ASC, id DESC)
+    `.execute();
+
+    await db.sql`
+      CREATE INDEX bucket_parameters_source_index ON bucket_parameters (group_id, source_table, source_key)
+    `.execute();
+
+    await db.sql`
+      CREATE TABLE current_data (
+        group_id integer NOT NULL,
+        source_table TEXT NOT NULL,
+        source_key bytea NOT NULL,
+        CONSTRAINT unique_current_data_id PRIMARY KEY (group_id, source_table, source_key),
+        buckets jsonb NOT NULL,
+        data bytea NOT NULL,
+        lookups bytea[] NOT NULL
+      );
+    `.execute();
+
+    await db.sql`
+      CREATE TABLE source_tables (
+        --- This is currently a TEXT column to make the (shared) tests easier to integrate
+        --- we could improve this if necessary
+        id TEXT PRIMARY KEY,
+        group_id integer NOT NULL,
+        connection_id integer NOT NULL,
+        relation_id jsonb,
+        schema_name text NOT NULL,
+        table_name text NOT NULL,
+        replica_id_columns jsonb,
+        snapshot_done BOOLEAN NOT NULL DEFAULT FALSE
+      )
+    `.execute();
+
+    await db.sql`CREATE INDEX source_table_lookup ON source_tables (group_id, table_name)`.execute();
+
+    await db.sql`
+      CREATE TABLE write_checkpoints (
+        user_id text PRIMARY KEY,
+        lsns jsonb NOT NULL,
+        write_checkpoint BIGINT NOT NULL
+      )
+    `.execute();
+
+    await db.sql`
+      CREATE TABLE custom_write_checkpoints (
+        user_id text NOT NULL,
+        write_checkpoint BIGINT NOT NULL,
+        sync_rules_id integer NOT NULL,
+        CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id)
+      );
+    `.execute();
+  });
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  await dropTables(client);
+};

package/src/module/PostgresStorageModule.ts

@@ -0,0 +1,30 @@
+import { modules, system } from '@powersync/service-core';
+
+import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
+import { PostgresStorageProvider } from '../storage/PostgresStorageProvider.js';
+import { isPostgresStorageConfig, PostgresStorageConfig } from '../types/types.js';
+
+export class PostgresStorageModule extends modules.AbstractModule {
+  constructor() {
+    super({
+      name: 'Postgres Bucket Storage'
+    });
+  }
+
+  async initialize(context: system.ServiceContextContainer): Promise<void> {
+    const { storageEngine } = context;
+
+    // Register the ability to use Postgres as a BucketStorage
+    storageEngine.registerProvider(new PostgresStorageProvider());
+
+    if (isPostgresStorageConfig(context.configuration.storage)) {
+      context.migrations.registerMigrationAgent(
+        new PostgresMigrationAgent(PostgresStorageConfig.decode(context.configuration.storage))
+      );
+    }
+  }
+
+  async teardown(): Promise<void> {
+    // Teardown for this module is implemented in the storage engine
+  }
+}