@powersync/service-module-postgres 0.13.0 → 0.14.0

package/src/replication/WalStreamReplicationJob.ts

@@ -1,4 +1,4 @@
-import { container } from '@powersync/lib-services-framework';
+import { container, logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
 import { PgManager } from './PgManager.js';
 import { MissingReplicationSlotError, sendKeepAlive, WalStream } from './WalStream.js';
 
@@ -12,9 +12,11 @@ export interface WalStreamReplicationJobOptions extends replication.AbstractRepl
 export class WalStreamReplicationJob extends replication.AbstractReplicationJob {
   private connectionFactory: ConnectionManagerFactory;
   private readonly connectionManager: PgManager;
+  private lastStream: WalStream | null = null;
 
   constructor(options: WalStreamReplicationJobOptions) {
     super(options);
+    this.logger = logger.child({ prefix: `[${this.slotName}] ` });
     this.connectionFactory = options.connectionFactory;
     this.connectionManager = this.connectionFactory.create({
       // Pool connections are only used intermittently.
@@ -57,7 +59,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
           replication_slot: this.slotName
         }
       });
-      this.logger.error(`Replication failed on ${this.slotName}`, e);
+      this.logger.error(`Replication failed`, e);
 
       if (e instanceof MissingReplicationSlotError) {
         // This stops replication on this slot and restarts with a new slot
@@ -93,14 +95,20 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
         return;
       }
       const stream = new WalStream({
+        logger: this.logger,
         abort_signal: this.abortController.signal,
         storage: this.options.storage,
         metrics: this.options.metrics,
         connections: connectionManager
       });
+      this.lastStream = stream;
       await stream.replicate();
     } catch (e) {
-      this.logger.error(`${this.slotName} Replication error`, e);
+      if (this.isStopped && e instanceof ReplicationAbortedError) {
+        // Ignore aborted errors
+        return;
+      }
+      this.logger.error(`Replication error`, e);
       if (e.cause != null) {
         // Example:
         // PgError.conn_ended: Unable to do postgres query on ended connection
@@ -140,4 +148,8 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
       await connectionManager.end();
     }
   }
+
+  async getReplicationLagMillis(): Promise<number | undefined> {
+    return this.lastStream?.getReplicationLagMillis();
+  }
 }

package/src/replication/WalStreamReplicator.ts

@@ -48,4 +48,30 @@ export class WalStreamReplicator extends replication.AbstractReplicator<WalStrea
   async testConnection() {
     return await PostgresModule.testConnection(this.connectionFactory.dbConnectionConfig);
   }
+
+  async getReplicationLagMillis(): Promise<number | undefined> {
+    const lag = await super.getReplicationLagMillis();
+    if (lag != null) {
+      return lag;
+    }
+
+    // Booting or in an error loop. Check last active replication status.
+    // This includes sync rules in an ERROR state.
+    const content = await this.storage.getActiveSyncRulesContent();
+    if (content == null) {
+      return undefined;
+    }
+    // Measure the lag from the last commit or keepalive timestamp.
+    // This is not 100% accurate since it is the commit time in the storage db rather than
+    // the source db, but it's the best we have for postgres.
+
+    const checkpointTs = content.last_checkpoint_ts?.getTime() ?? 0;
+    const keepaliveTs = content.last_keepalive_ts?.getTime() ?? 0;
+    const latestTs = Math.max(checkpointTs, keepaliveTs);
+    if (latestTs != 0) {
+      return Date.now() - latestTs;
+    }
+
+    return undefined;
+  }
 }

package/src/replication/replication-utils.ts

@@ -1,7 +1,7 @@
 import * as pgwire from '@powersync/service-jpgwire';
 
 import * as lib_postgres from '@powersync/lib-service-postgres';
-import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
 import { PatternResult, storage } from '@powersync/service-core';
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as service_types from '@powersync/service-types';
@@ -136,6 +136,61 @@ $$ LANGUAGE plpgsql;`
   }
 }
 
+export async function checkTableRls(
+  db: pgwire.PgClient,
+  relationId: number
+): Promise<{ canRead: boolean; message?: string }> {
+  const rs = await lib_postgres.retriedQuery(db, {
+    statement: `
+WITH user_info AS (
+    SELECT 
+        current_user as username,
+        r.rolsuper,
+        r.rolbypassrls
+    FROM pg_roles r 
+    WHERE r.rolname = current_user
+)
+SELECT
+    c.relname as tablename,
+    c.relrowsecurity as rls_enabled,
+    u.username as username,
+    u.rolsuper as is_superuser,
+    u.rolbypassrls as bypasses_rls
+FROM pg_class c
+CROSS JOIN user_info u
+WHERE c.oid = $1::oid;
+`,
+    params: [{ type: 'int4', value: relationId }]
+  });
+
+  const rows = pgwire.pgwireRows<{
+    rls_enabled: boolean;
+    tablename: string;
+    username: string;
+    is_superuser: boolean;
+    bypasses_rls: boolean;
+  }>(rs);
+  if (rows.length == 0) {
+    // Not expected, since we already got the oid
+    throw new ServiceAssertionError(`Table with OID ${relationId} does not exist.`);
+  }
+  const row = rows[0];
+  if (row.is_superuser || row.bypasses_rls) {
+    // Bypasses RLS automatically.
+    return { canRead: true };
+  }
+
+  if (row.rls_enabled) {
+    // Don't skip, since we _may_ still be able to get results.
+    return {
+      canRead: false,
+      message: `[${ErrorCode.PSYNC_S1145}] Row Level Security is enabled on table "${row.tablename}". To make sure that ${row.username} can read the table, run: 'ALTER ROLE ${row.username} BYPASSRLS'.`
+    };
+  }
+
+  return { canRead: true };
+}
+
 export interface GetDebugTablesInfoOptions {
   db: pgwire.PgClient;
   publicationName: string;
@@ -309,6 +364,9 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom
     };
   }
 
+  const rlsCheck = await checkTableRls(db, relationId);
+  const rlsError = rlsCheck.canRead ? null : { message: rlsCheck.message!, level: 'warning' };
+
   return {
     schema: schema,
     name: name,
@@ -316,7 +374,7 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom
     replication_id: id_columns.map((c) => c.name),
     data_queries: syncData,
     parameter_queries: syncParameters,
-    errors: [id_columns_error, selectError, replicateError].filter(
+    errors: [id_columns_error, selectError, replicateError, rlsError].filter(
       (error) => error != null
     ) as service_types.ReplicationError[]
   };

package/test/src/__snapshots__/schema_changes.test.ts.snap

@@ -1,5 +1,5 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`schema changes - mongodb > add to publication (not in sync rules) 1`] = `0`;
+exports[`schema changes > mongodb storage > add to publication (not in sync rules) 1`] = `0`;
 
-exports[`schema changes - postgres > add to publication (not in sync rules) 1`] = `16384`;
+exports[`schema changes > postgres storage > add to publication (not in sync rules) 1`] = `16384`;

package/test/src/checkpoints.test.ts

@@ -1,9 +1,8 @@
 import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js';
-import { checkpointUserId, createWriteCheckpoint } from '@powersync/service-core';
+import { checkpointUserId, createWriteCheckpoint, TestStorageFactory } from '@powersync/service-core';
 import { describe, test } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { describeWithStorage } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
-import { env } from './env.js';
 
 import timers from 'node:timers/promises';
 
@@ -12,14 +11,25 @@ const BASIC_SYNC_RULES = `bucket_definitions:
     data:
       - SELECT id, description, other FROM "test_data"`;
 
-describe.skipIf(!(env.CI || env.SLOW_TESTS))('checkpoint tests', () => {
+describe('checkpoint tests', () => {
+  describeWithStorage({}, checkpointTests);
+});
+
+const checkpointTests = (factory: TestStorageFactory) => {
   test('write checkpoints', { timeout: 50_000 }, async () => {
-    const factory = INITIALIZED_MONGO_STORAGE_FACTORY;
     await using context = await WalStreamTestContext.open(factory);
 
     await context.updateSyncRules(BASIC_SYNC_RULES);
     const { pool } = context;
     const api = new PostgresRouteAPIAdapter(pool);
+    const serverVersion = await context.connectionManager.getServerVersion();
+    if (serverVersion!.compareMain('13.0.0') < 0) {
+      // The test is not stable on Postgres 11 or 12. See the notes on
+      // PostgresRouteAPIAdapter.createReplicationHead() for details.
+      // Postgres 12 is already EOL, so not worth finding a fix - just skip the tests.
+      console.log('Skipping write checkpoint test on Postgres < 13.0.0');
+      return;
+    }
 
     await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
 
@@ -59,7 +69,7 @@ describe.skipIf(!(env.CI || env.SLOW_TESTS))('checkpoint tests', () => {
 
         const start = Date.now();
         while (lastWriteCheckpoint == null || lastWriteCheckpoint < BigInt(cp.writeCheckpoint)) {
-          if (Date.now() - start > 5_000) {
+          if (Date.now() - start > 3_000) {
             throw new Error(
               `Timeout while waiting for checkpoint. last: ${lastWriteCheckpoint}, waiting for: ${cp.writeCheckpoint}`
             );
@@ -71,4 +81,4 @@ describe.skipIf(!(env.CI || env.SLOW_TESTS))('checkpoint tests', () => {
       controller.abort();
     }
   });
-});
+};

package/test/src/chunked_snapshots.test.ts

@@ -0,0 +1,156 @@
+import { reduceBucket, TestStorageFactory } from '@powersync/service-core';
+import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as crypto from 'node:crypto';
+import * as timers from 'timers/promises';
+import { describe, expect, test } from 'vitest';
+import { describeWithStorage } from './util.js';
+import { WalStreamTestContext } from './wal_stream_utils.js';
+
+describe('chunked snapshots', () => {
+  describeWithStorage({ timeout: 120_000 }, defineBatchTests);
+});
+
+function defineBatchTests(factory: TestStorageFactory) {
+  // We need to test every supported type, since chunking could be quite sensitive to
+  // how each specific type is handled.
+  test('chunked snapshot edge case (int2)', async () => {
+    await testChunkedSnapshot({
+      idType: 'int2',
+      genId: 'i',
+      lastId: '2000',
+      moveTo: '0',
+      moveToJs: 0
+    });
+  });
+
+  test('chunked snapshot edge case (int4)', async () => {
+    await testChunkedSnapshot({
+      idType: 'int4',
+      genId: 'i',
+      lastId: '2000',
+      moveTo: '0',
+      moveToJs: 0
+    });
+  });
+
+  test('chunked snapshot edge case (int8)', async () => {
+    await testChunkedSnapshot({
+      idType: 'int8',
+      genId: 'i',
+      lastId: '2000',
+      moveTo: '0',
+      moveToJs: 0
+    });
+  });
+
+  test('chunked snapshot edge case (text)', async () => {
+    await testChunkedSnapshot({
+      idType: 'text',
+      genId: `to_char(i, 'fm0000')`,
+      lastId: `'2000'`,
+      moveTo: `'0000'`,
+      moveToJs: '0000'
+    });
+  });
+
+  test('chunked snapshot edge case (varchar)', async () => {
+    await testChunkedSnapshot({
+      idType: 'varchar',
+      genId: `to_char(i, 'fm0000')`,
+      lastId: `'2000'`,
+      moveTo: `'0000'`,
+      moveToJs: '0000'
+    });
+  });
+
+  test('chunked snapshot edge case (uuid)', async () => {
+    await testChunkedSnapshot({
+      idType: 'uuid',
+      // Generate a uuid by using the first part of a uuid and appending a 4-digit number.
+      genId: `('00000000-0000-4000-8000-00000000' || to_char(i, 'fm0000')) :: uuid`,
+      lastId: `'00000000-0000-4000-8000-000000002000'`,
+      moveTo: `'00000000-0000-4000-8000-000000000000'`,
+      moveToJs: '00000000-0000-4000-8000-000000000000'
+    });
+  });
+
+  async function testChunkedSnapshot(options: {
+    idType: string;
+    genId: string;
+    lastId: string;
+    moveTo: string;
+    moveToJs: SqliteJsonValue;
+  }) {
+    // 1. Start with 2k rows, one row with id = 2000, and a large TOAST value in another column.
+    // 2. Replicate one batch of rows (id < 2000).
+    // 3. `UPDATE table SET id = 0 WHERE id = 2000`
+    // 4. Replicate the rest of the table.
+    // 5. Logical replication picks up the UPDATE above, but it is missing the TOAST column.
+    // 6. We end up with a row that has a missing TOAST column.
+
+    await using context = await WalStreamTestContext.open(factory, {
+      // We need to use a smaller chunk size here, so that we can run a query in between chunks
+      walStreamOptions: { snapshotChunkLength: 100 }
+    });
+
+    await context.updateSyncRules(`bucket_definitions:
+  global:
+    data:
+      - SELECT * FROM test_data`);
+    const { pool } = context;
+
+    await pool.query(`CREATE TABLE test_data(id ${options.idType} primary key, description text)`);
+
+    // 1. Start with 2k rows, one row with id = 2000...
+    await pool.query({
+      statement: `INSERT INTO test_data(id, description) SELECT ${options.genId}, 'foo' FROM generate_series(1, 2000) i`
+    });
+
+    // ...and a large TOAST value in another column.
+    // Toast value, must be > 8kb after compression
+    const largeDescription = crypto.randomBytes(20_000).toString('hex');
+    await pool.query({
+      statement: `UPDATE test_data SET description = $1 WHERE id = ${options.lastId} :: ${options.idType}`,
+      params: [{ type: 'varchar', value: largeDescription }]
+    });
+
+    // 2. Replicate one batch of rows (id < 100).
+    // Our "stopping point" here is not quite deterministic.
+    const p = context.replicateSnapshot();
+
+    const stopAfter = 100;
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+
+    while (true) {
+      const count =
+        ((await METRICS_HELPER.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0) - startRowCount;
+
+      if (count >= stopAfter) {
+        break;
+      }
+      await timers.setTimeout(1);
+    }
+
+    // 3. `UPDATE table SET id = 0 WHERE id = 2000`
+    const rs = await pool.query(
+      `UPDATE test_data SET id = ${options.moveTo} WHERE id = ${options.lastId} RETURNING id`
+    );
+    expect(rs.rows.length).toEqual(1);
+
+    // 4. Replicate the rest of the table.
+    await p;
+
+    // 5. Logical replication picks up the UPDATE above, but it is missing the TOAST column.
+    context.startStreaming();
+
+    // 6. If all went well, the "resnapshot" process would take care of this.
+    const data = await context.getBucketData('global[]', undefined, {});
+    const reduced = reduceBucket(data);
+
+    const movedRow = reduced.find((row) => row.object_id == String(options.moveToJs));
+    expect(movedRow?.data).toEqual(JSON.stringify({ id: options.moveToJs, description: largeDescription }));
+
+    expect(reduced.length).toEqual(2001);
+  }
+}

package/test/src/large_batch.test.ts

@@ -1,37 +1,14 @@
 import { storage } from '@powersync/service-core';
-import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 import { populateData } from '../../dist/utils/populate_test_data.js';
 import { env } from './env.js';
-import {
-  INITIALIZED_MONGO_STORAGE_FACTORY,
-  INITIALIZED_POSTGRES_STORAGE_FACTORY,
-  TEST_CONNECTION_OPTIONS
-} from './util.js';
+import { describeWithStorage, TEST_CONNECTION_OPTIONS } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
-import { METRICS_HELPER } from '@powersync/service-core-tests';
-import { ReplicationMetric } from '@powersync/service-types';
-
-describe.skipIf(!env.TEST_MONGO_STORAGE)('batch replication tests - mongodb', { timeout: 120_000 }, function () {
-  // These are slow but consistent tests.
-  // Not run on every test run, but we do run on CI, or when manually debugging issues.
-  if (env.CI || env.SLOW_TESTS) {
-    defineBatchTests(INITIALIZED_MONGO_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
-});
 
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)('batch replication tests - postgres', { timeout: 240_000 }, function () {
-  // These are slow but consistent tests.
-  // Not run on every test run, but we do run on CI, or when manually debugging issues.
-  if (env.CI || env.SLOW_TESTS) {
-    defineBatchTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
+describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
+  describeWithStorage({ timeout: 240_000 }, function (factory) {
+    defineBatchTests(factory);
+  });
 });
 
 const BASIC_SYNC_RULES = `bucket_definitions:
@@ -257,132 +234,6 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
     expect(checksum.get('global[]')!.count).toEqual((numDocs + 2) * 4);
   });
 
-  test('resuming initial replication (1)', async () => {
-    // Stop early - likely to not include deleted row in first replication attempt.
-    await testResumingReplication(2000);
-  });
-  test('resuming initial replication (2)', async () => {
-    // Stop late - likely to include deleted row in first replication attempt.
-    await testResumingReplication(8000);
-  });
-
-  async function testResumingReplication(stopAfter: number) {
-    // This tests interrupting and then resuming initial replication.
-    // We interrupt replication after test_data1 has fully replicated, and
-    // test_data2 has partially replicated.
-    // This test relies on interval behavior that is not 100% deterministic:
-    // 1. We attempt to abort initial replication once a certain number of
-    //    rows have been replicated, but this is not exact. Our only requirement
-    //    is that we have not fully replicated test_data2 yet.
-    // 2. Order of replication is not deterministic, so which specific rows
-    //    have been / have not been replicated at that point is not deterministic.
-    //    We do allow for some variation in the test results to account for this.
-
-    await using context = await WalStreamTestContext.open(factory);
-
-    await context.updateSyncRules(`bucket_definitions:
-  global:
-    data:
-      - SELECT * FROM test_data1
-      - SELECT * FROM test_data2`);
-    const { pool } = context;
-
-    await pool.query(`CREATE TABLE test_data1(id serial primary key, description text)`);
-    await pool.query(`CREATE TABLE test_data2(id serial primary key, description text)`);
-
-    await pool.query(
-      {
-        statement: `INSERT INTO test_data1(description) SELECT 'foo' FROM generate_series(1, 1000) i`
-      },
-      {
-        statement: `INSERT INTO test_data2( description) SELECT 'foo' FROM generate_series(1, 10000) i`
-      }
-    );
-
-    const p = context.replicateSnapshot();
-
-    let done = false;
-
-    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
-    try {
-      (async () => {
-        while (!done) {
-          const count =
-            ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
-
-          if (count >= stopAfter) {
-            break;
-          }
-          await timers.setTimeout(1);
-        }
-        // This interrupts initial replication
-        await context.dispose();
-      })();
-      // This confirms that initial replication was interrupted
-      await expect(p).rejects.toThrowError();
-      done = true;
-    } finally {
-      done = true;
-    }
-
-    // Bypass the usual "clear db on factory open" step.
-    await using context2 = await WalStreamTestContext.open(factory, { doNotClear: true });
-
-    // This delete should be using one of the ids already replicated
-    const {
-      rows: [[id1]]
-    } = await context2.pool.query(`DELETE FROM test_data2 WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id`);
-    // This update should also be using one of the ids already replicated
-    const {
-      rows: [[id2]]
-    } = await context2.pool.query(
-      `UPDATE test_data2 SET description = 'update1' WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id`
-    );
-    const {
-      rows: [[id3]]
-    } = await context2.pool.query(`INSERT INTO test_data2(description) SELECT 'insert1' RETURNING id`);
-
-    await context2.loadNextSyncRules();
-    await context2.replicateSnapshot();
-
-    context2.startStreaming();
-    const data = await context2.getBucketData('global[]', undefined, {});
-
-    const deletedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === String(id1));
-    const updatedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === String(id2));
-    const insertedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === String(id3));
-
-    if (deletedRowOps.length != 0) {
-      // The deleted row was part of the first replication batch,
-      // so it is removed by streaming replication.
-      expect(deletedRowOps.length).toEqual(2);
-      expect(deletedRowOps[1].op).toEqual('REMOVE');
-    } else {
-      // The deleted row was not part of the first replication batch,
-      // so it's not in the resulting ops at all.
-    }
-
-    expect(updatedRowOps.length).toEqual(2);
-    // description for the first op could be 'foo' or 'update1'.
-    // We only test the final version.
-    expect(JSON.parse(updatedRowOps[1].data as string).description).toEqual('update1');
-
-    expect(insertedRowOps.length).toEqual(2);
-    expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1');
-    expect(JSON.parse(insertedRowOps[1].data as string).description).toEqual('insert1');
-
-    // 1000 of test_data1 during first replication attempt.
-    // N >= 1000 of test_data2 during first replication attempt.
-    // 10000 - N - 1 + 1 of test_data2 during second replication attempt.
-    // An additional update during streaming replication (2x total for this row).
-    // An additional insert during streaming replication (2x total for this row).
-    // If the deleted row was part of the first replication batch, it's removed by streaming replication.
-    // This adds 2 ops.
-    // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000.
-    // However, this is not deterministic.
-    expect(data.length).toEqual(11002 + deletedRowOps.length);
-  }
-
   function printMemoryUsage() {
     const memoryUsage = process.memoryUsage();