@powersync/service-module-postgres 0.0.0-dev-20240918092408

package/test/src/slow_tests.test.ts

@@ -0,0 +1,335 @@
+import * as bson from 'bson';
+import { afterEach, describe, expect, test } from 'vitest';
+import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
+import { env } from './env.js';
+import { TEST_CONNECTION_OPTIONS, clearTestDb, connectPgPool, getClientCheckpoint } from './util.js';
+
+import * as pgwire from '@powersync/service-jpgwire';
+import { SqliteRow } from '@powersync/service-sync-rules';
+
+import { mapOpEntry, MongoBucketStorage } from '@/storage/storage-index.js';
+import * as timers from 'node:timers/promises';
+import { MONGO_STORAGE_FACTORY, StorageFactory } from '@core-tests/util.js';
+import { PgManager } from '@module/replication/PgManager.js';
+import { reduceBucket, validateCompactedBucket } from '@core-tests/bucket_validation.js';
+
+describe('slow tests - mongodb', function () {
+  // These are slow, inconsistent tests.
+  // Not run on every test run, but we do run on CI, or when manually debugging issues.
+  if (env.CI || env.SLOW_TESTS) {
+    defineSlowTests(MONGO_STORAGE_FACTORY);
+  } else {
+    // Need something in this file.
+    test('no-op', () => {});
+  }
+});
+
+function defineSlowTests(factory: StorageFactory) {
+  let walStream: WalStream | undefined;
+  let connections: PgManager | undefined;
+  let abortController: AbortController | undefined;
+  let streamPromise: Promise<void> | undefined;
+
+  afterEach(async () => {
+    // This cleans up, similar to WalStreamTestContext.dispose().
+    // These tests are a little more complex than what is supported by WalStreamTestContext.
+    abortController?.abort();
+    await streamPromise;
+    streamPromise = undefined;
+    connections?.destroy();
+
+    connections = undefined;
+    walStream = undefined;
+    abortController = undefined;
+  });
+
+  const TEST_DURATION_MS = 15_000;
+  const TIMEOUT_MARGIN_MS = env.CI ? 30_000 : 15_000;
+
+  // Test repeatedly replicating inserts and deletes, then check that we get
+  // consistent data out at the end.
+  //
+  // Past issues that this could reproduce intermittently:
+  // * Skipping LSNs after a keepalive message
+  // * Skipping LSNs when source transactions overlap
+  test(
+    'repeated replication - basic',
+    async () => {
+      await testRepeatedReplication({ compact: false, maxBatchSize: 50, numBatches: 5 });
+    },
+    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
+  );
+
+  test(
+    'repeated replication - compacted',
+    async () => {
+      await testRepeatedReplication({ compact: true, maxBatchSize: 100, numBatches: 2 });
+    },
+    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
+  );
+
+  async function testRepeatedReplication(testOptions: { compact: boolean; maxBatchSize: number; numBatches: number }) {
+    const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
+    const replicationConnection = await connections.replicationConnection();
+    const pool = connections.pool;
+    await clearTestDb(pool);
+    const f = (await factory()) as MongoBucketStorage;
+
+    const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT * FROM "test_data"
+`;
+    const syncRules = await f.updateSyncRules({ content: syncRuleContent });
+    const storage = f.getInstance(syncRules);
+    abortController = new AbortController();
+    const options: WalStreamOptions = {
+      abort_signal: abortController.signal,
+      connections,
+      storage: storage
+    };
+    walStream = new WalStream(options);
+
+    await pool.query(
+      `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num decimal)`
+    );
+    await pool.query(`ALTER TABLE test_data REPLICA IDENTITY FULL`);
+
+    await walStream.initReplication(replicationConnection);
+    await storage.autoActivate();
+    let abort = false;
+    streamPromise = walStream.streamChanges(replicationConnection).finally(() => {
+      abort = true;
+    });
+    const start = Date.now();
+
+    while (!abort && Date.now() - start < TEST_DURATION_MS) {
+      const bg = async () => {
+        for (let j = 0; j < testOptions.numBatches && !abort; j++) {
+          const n = Math.max(1, Math.floor(Math.random() * testOptions.maxBatchSize));
+          let statements: pgwire.Statement[] = [];
+          for (let i = 0; i < n; i++) {
+            const description = `test${i}`;
+            statements.push({
+              statement: `INSERT INTO test_data(description, num) VALUES($1, $2) returning id as test_id`,
+              params: [
+                { type: 'varchar', value: description },
+                { type: 'float8', value: Math.random() }
+              ]
+            });
+          }
+          const results = await pool.query(...statements);
+          const ids = results.results.map((sub) => {
+            return sub.rows[0][0] as string;
+          });
+          await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
+
+          if (Math.random() > 0.5) {
+            const updateStatements: pgwire.Statement[] = ids.map((id) => {
+              return {
+                statement: `UPDATE test_data SET num = $2 WHERE id = $1`,
+                params: [
+                  { type: 'uuid', value: id },
+                  { type: 'float8', value: Math.random() }
+                ]
+              };
+            });
+
+            await pool.query(...updateStatements);
+            if (Math.random() > 0.5) {
+              // Special case - an update that doesn't change data
+              await pool.query(...updateStatements);
+            }
+          }
+
+          const deleteStatements: pgwire.Statement[] = ids.map((id) => {
+            return {
+              statement: `DELETE FROM test_data WHERE id = $1`,
+              params: [{ type: 'uuid', value: id }]
+            };
+          });
+          await pool.query(...deleteStatements);
+
+          await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
+        }
+      };
+
+      let compactController = new AbortController();
+
+      const bgCompact = async () => {
+        // Repeatedly compact, and check that the compact conditions hold
+        while (!compactController.signal.aborted) {
+          const delay = Math.random() * 50;
+          try {
+            await timers.setTimeout(delay, undefined, { signal: compactController.signal });
+          } catch (e) {
+            break;
+          }
+
+          const checkpoint = BigInt((await storage.getCheckpoint()).checkpoint);
+          const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+            .filter((row) => row._id.o <= checkpoint)
+            .map(mapOpEntry);
+          await storage.compact({ maxOpId: checkpoint });
+          const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+            .filter((row) => row._id.o <= checkpoint)
+            .map(mapOpEntry);
+
+          validateCompactedBucket(opsBefore, opsAfter);
+        }
+      };
+
+      // Call the above loop multiple times concurrently
+      const promises = [1, 2, 3].map((i) => bg());
+      const compactPromise = testOptions.compact ? bgCompact() : null;
+      await Promise.all(promises);
+      compactController.abort();
+      await compactPromise;
+
+      // Wait for replication to finish
+      let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS });
+
+      // Check that all inserts have been deleted again
+      const docs = await f.db.current_data.find().toArray();
+      const transformed = docs.map((doc) => {
+        return bson.deserialize(doc.data.buffer) as SqliteRow;
+      });
+      expect(transformed).toEqual([]);
+
+      // Check that each PUT has a REMOVE
+      const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
+
+      // All a single bucket in this test
+      const bucket = ops.map((op) => mapOpEntry(op));
+      const reduced = reduceBucket(bucket);
+      expect(reduced).toMatchObject([
+        {
+          op_id: '0',
+          op: 'CLEAR'
+        }
+        // Should contain no additional data
+      ]);
+    }
+
+    abortController.abort();
+    await streamPromise;
+  }
+
+  // Test repeatedly performing initial replication.
+  //
+  // If the first LSN does not correctly match with the first replication transaction,
+  // we may miss some updates.
+  test(
+    'repeated initial replication',
+    async () => {
+      const pool = await connectPgPool();
+      await clearTestDb(pool);
+      const f = await factory();
+
+      const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+`;
+      const syncRules = await f.updateSyncRules({ content: syncRuleContent });
+      const storage = f.getInstance(syncRules);
+
+      // 1. Setup some base data that will be replicated in initial replication
+      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      let statements: pgwire.Statement[] = [];
+
+      const n = Math.floor(Math.random() * 200);
+      for (let i = 0; i < n; i++) {
+        statements.push({
+          statement: `INSERT INTO test_data(description) VALUES('test_init')`
+        });
+      }
+      await pool.query(...statements);
+
+      const start = Date.now();
+      let i = 0;
+
+      while (Date.now() - start < TEST_DURATION_MS) {
+        // 2. Each iteration starts with a clean slate
+        await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`);
+        i += 1;
+
+        const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
+        const replicationConnection = await connections.replicationConnection();
+
+        abortController = new AbortController();
+        const options: WalStreamOptions = {
+          abort_signal: abortController.signal,
+          connections,
+          storage: storage
+        };
+        walStream = new WalStream(options);
+
+        await storage.clear();
+
+        // 3. Start initial replication, then streaming, but don't wait for any of this
+        let initialReplicationDone = false;
+        streamPromise = (async () => {
+          await walStream.initReplication(replicationConnection);
+          await storage.autoActivate();
+          initialReplicationDone = true;
+          await walStream.streamChanges(replicationConnection);
+        })()
+          .catch((e) => {
+            initialReplicationDone = true;
+            throw e;
+          })
+          .then((v) => {
+            return v;
+          });
+
+        // 4. While initial replication is still running, write more changes
+        while (!initialReplicationDone) {
+          let statements: pgwire.Statement[] = [];
+          const n = Math.floor(Math.random() * 10) + 1;
+          for (let i = 0; i < n; i++) {
+            const description = `test${i}`;
+            statements.push({
+              statement: `INSERT INTO test_data(description) VALUES('test1') returning id as test_id`,
+              params: [{ type: 'varchar', value: description }]
+            });
+          }
+          const results = await pool.query(...statements);
+          const ids = results.results.map((sub) => {
+            return sub.rows[0][0] as string;
+          });
+          await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
+          const deleteStatements: pgwire.Statement[] = ids.map((id) => {
+            return {
+              statement: `DELETE FROM test_data WHERE id = $1`,
+              params: [{ type: 'uuid', value: id }]
+            };
+          });
+          await pool.query(...deleteStatements);
+          await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
+        }
+
+        // 5. Once initial replication is done, wait for the streaming changes to complete syncing.
+        // getClientCheckpoint() effectively waits for the above replication to complete
+        // Race with streamingPromise to catch replication errors here.
+        let checkpoint = await Promise.race([
+          getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }),
+          streamPromise
+        ]);
+        if (typeof checkpoint == undefined) {
+          // This indicates an issue with the test setup - streamingPromise completed instead
+          // of getClientCheckpoint()
+          throw new Error('Test failure - streamingPromise completed');
+        }
+
+        abortController.abort();
+        await streamPromise;
+        await connections.end();
+      }
+    },
+    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
+  );
+}

package/test/src/util.ts

@@ -0,0 +1,105 @@
+import * as types from '@module/types/types.js';
+import * as pg_utils from '@module/utils/pgwire_utils.js';
+import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId } from '@powersync/service-core';
+import * as pgwire from '@powersync/service-jpgwire';
+import { env } from './env.js';
+import { pgwireRows } from '@powersync/service-jpgwire';
+import { logger } from '@powersync/lib-services-framework';
+import { connectMongo } from '@core-tests/util.js';
+
+// The metrics need to be initialized before they can be used
+await Metrics.initialise({
+  disable_telemetry_sharing: true,
+  powersync_instance_id: 'test',
+  internal_metrics_endpoint: 'unused.for.tests.com'
+});
+Metrics.getInstance().resetCounters();
+
+export const TEST_URI = env.PG_TEST_URL;
+
+export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
+  type: 'postgresql',
+  uri: TEST_URI,
+  sslmode: 'disable'
+});
+
+export type StorageFactory = () => Promise<BucketStorageFactory>;
+
+export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => {
+  const db = await connectMongo();
+
+  // None of the PG tests insert data into this collection, so it was never created
+  if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+    await db.db.createCollection('bucket_parameters');
+  }
+
+  await db.clear();
+
+  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
+};
+
+export async function clearTestDb(db: pgwire.PgClient) {
+  await db.query(
+    "select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false and slot_name like 'test_%'"
+  );
+
+  await db.query(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
+  try {
+    await db.query(`DROP PUBLICATION powersync`);
+  } catch (e) {
+    // Ignore
+  }
+
+  await db.query(`CREATE PUBLICATION powersync FOR ALL TABLES`);
+
+  const tableRows = pgwire.pgwireRows(
+    await db.query(`SELECT table_name FROM information_schema.tables where table_schema = 'public'`)
+  );
+  for (let row of tableRows) {
+    const name = row.table_name;
+    if (name.startsWith('test_')) {
+      await db.query(`DROP TABLE public.${pg_utils.escapeIdentifier(name)}`);
+    }
+  }
+}
+
+export async function connectPgWire(type?: 'replication' | 'standard') {
+  const db = await pgwire.connectPgWire(TEST_CONNECTION_OPTIONS, { type });
+  return db;
+}
+
+export function connectPgPool() {
+  const db = pgwire.connectPgWirePool(TEST_CONNECTION_OPTIONS);
+  return db;
+}
+
+export async function getClientCheckpoint(
+  db: pgwire.PgClient,
+  bucketStorage: BucketStorageFactory,
+  options?: { timeout?: number }
+): Promise<OpId> {
+  const start = Date.now();
+
+  const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
+
+  // This old API needs a persisted checkpoint id.
+  // Since we don't use LSNs anymore, the only way to get that is to wait.
+
+  const timeout = options?.timeout ?? 50_000;
+
+  logger.info(`Waiting for LSN checkpoint: ${lsn}`);
+  while (Date.now() - start < timeout) {
+    const cp = await bucketStorage.getActiveCheckpoint();
+    if (!cp.hasSyncRules()) {
+      throw new Error('No sync rules available');
+    }
+    if (cp.lsn && cp.lsn >= lsn) {
+      logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
+      return cp.checkpoint;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 30));
+  }
+
+  throw new Error('Timeout while waiting for checkpoint');
+}

package/test/src/validation.test.ts

@@ -0,0 +1,64 @@
+import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
+import { expect, test } from 'vitest';
+import { walStreamTest } from './wal_stream_utils.js';
+import { getDebugTablesInfo } from '@module/replication/replication-utils.js';
+
+// Not quite a walStreamTest, but it helps to manage the connection
+test(
+  'validate tables',
+  walStreamTest(MONGO_STORAGE_FACTORY, async (context) => {
+    const { pool } = context;
+
+    await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+    const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+      - SELECT * FROM "other"
+      - SELECT * FROM "other%"
+`;
+
+    const syncRules = await context.factory.updateSyncRules({ content: syncRuleContent });
+
+    const tablePatterns = syncRules.parsed({ defaultSchema: 'public' }).sync_rules.getSourceTables();
+    const tableInfo = await getDebugTablesInfo({
+      db: pool,
+      publicationName: context.publicationName,
+      connectionTag: context.connectionTag,
+      tablePatterns: tablePatterns,
+      syncRules: syncRules.parsed({ defaultSchema: 'public' }).sync_rules
+    });
+    expect(tableInfo).toEqual([
+      {
+        schema: 'public',
+        pattern: 'test_data',
+        wildcard: false,
+        table: {
+          schema: 'public',
+          name: 'test_data',
+          replication_id: ['id'],
+          pattern: undefined,
+          data_queries: true,
+          parameter_queries: false,
+          errors: []
+        }
+      },
+      {
+        schema: 'public',
+        pattern: 'other',
+        wildcard: false,
+        table: {
+          schema: 'public',
+          name: 'other',
+          replication_id: [],
+          data_queries: true,
+          parameter_queries: false,
+          errors: [{ level: 'warning', message: 'Table "public"."other" not found.' }]
+        }
+      },
+      { schema: 'public', pattern: 'other%', wildcard: true, tables: [] }
+    ]);
+  })
+);