@powersync/service-module-postgres 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/test/src/slow_tests.test.ts

@@ -1,412 +0,0 @@
-import * as bson from 'bson';
-import { afterEach, beforeAll, describe, expect, test } from 'vitest';
-import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
-import { env } from './env.js';
-import {
-  clearTestDb,
-  connectPgPool,
-  describeWithStorage,
-  getClientCheckpoint,
-  StorageVersionTestContext,
-  TEST_CONNECTION_OPTIONS
-} from './util.js';
-
-import * as pgwire from '@powersync/service-jpgwire';
-import { SqliteRow } from '@powersync/service-sync-rules';
-
-import { PgManager } from '@module/replication/PgManager.js';
-import {
-  createCoreReplicationMetrics,
-  initializeCoreReplicationMetrics,
-  updateSyncRulesFromYaml
-} from '@powersync/service-core';
-import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
-import * as mongo_storage from '@powersync/service-module-mongodb-storage';
-import * as postgres_storage from '@powersync/service-module-postgres-storage';
-import * as timers from 'node:timers/promises';
-import { CustomTypeRegistry } from '@module/types/registry.js';
-
-describe.skipIf(!(env.CI || env.SLOW_TESTS))('slow tests', function () {
-  describeWithStorage({ timeout: 120_000 }, function ({ factory, storageVersion }) {
-    defineSlowTests({ factory, storageVersion });
-  });
-});
-
-function defineSlowTests({ factory, storageVersion }: StorageVersionTestContext) {
-  let walStream: WalStream | undefined;
-  let connections: PgManager | undefined;
-  let abortController: AbortController | undefined;
-  let streamPromise: Promise<void> | undefined;
-
-  beforeAll(async () => {
-    createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
-    initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
-  });
-
-  afterEach(async () => {
-    // This cleans up, similar to WalStreamTestContext.dispose().
-    // These tests are a little more complex than what is supported by WalStreamTestContext.
-    abortController?.abort();
-    await streamPromise;
-    streamPromise = undefined;
-    connections?.destroy();
-
-    connections = undefined;
-    walStream = undefined;
-    abortController = undefined;
-  });
-
-  const TEST_DURATION_MS = 15_000;
-  const TIMEOUT_MARGIN_MS = env.CI ? 30_000 : 15_000;
-
-  // Test repeatedly replicating inserts and deletes, then check that we get
-  // consistent data out at the end.
-  //
-  // Past issues that this could reproduce intermittently:
-  // * Skipping LSNs after a keepalive message
-  // * Skipping LSNs when source transactions overlap
-  test('repeated replication - basic', { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }, async () => {
-    await testRepeatedReplication({ compact: false, maxBatchSize: 50, numBatches: 5 });
-  });
-
-  test('repeated replication - compacted', { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }, async () => {
-    await testRepeatedReplication({ compact: true, maxBatchSize: 100, numBatches: 2 });
-  });
-
-  async function testRepeatedReplication(testOptions: { compact: boolean; maxBatchSize: number; numBatches: number }) {
-    const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-    const replicationConnection = await connections.replicationConnection();
-    const pool = connections.pool;
-    await clearTestDb(pool);
-    await using f = await factory();
-
-    const syncRuleContent = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT * FROM "test_data"
-`;
-    const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion }));
-    const storage = f.getInstance(syncRules);
-    abortController = new AbortController();
-    const options: WalStreamOptions = {
-      abort_signal: abortController.signal,
-      connections,
-      storage: storage,
-      metrics: METRICS_HELPER.metricsEngine
-    };
-    walStream = new WalStream(options);
-
-    await pool.query(
-      `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num decimal)`
-    );
-    await pool.query(`ALTER TABLE test_data REPLICA IDENTITY FULL`);
-
-    await walStream.initReplication(replicationConnection);
-    let abort = false;
-    streamPromise = walStream.streamChanges(replicationConnection).finally(() => {
-      abort = true;
-    });
-    const start = Date.now();
-
-    while (!abort && Date.now() - start < TEST_DURATION_MS) {
-      const bg = async () => {
-        for (let j = 0; j < testOptions.numBatches && !abort; j++) {
-          const n = Math.max(1, Math.floor(Math.random() * testOptions.maxBatchSize));
-          let statements: pgwire.Statement[] = [];
-          for (let i = 0; i < n; i++) {
-            const description = `test${i}`;
-            statements.push({
-              statement: `INSERT INTO test_data(description, num) VALUES($1, $2) returning id as test_id`,
-              params: [
-                { type: 'varchar', value: description },
-                { type: 'float8', value: Math.random() }
-              ]
-            });
-          }
-          const results = await pool.query(...statements);
-          const ids = results.results.map((sub) => {
-            return sub.rows[0].decodeWithoutCustomTypes(0) as string;
-          });
-          await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
-
-          if (Math.random() > 0.5) {
-            const updateStatements: pgwire.Statement[] = ids.map((id) => {
-              return {
-                statement: `UPDATE test_data SET num = $2 WHERE id = $1`,
-                params: [
-                  { type: 'uuid', value: id },
-                  { type: 'float8', value: Math.random() }
-                ]
-              };
-            });
-
-            await pool.query(...updateStatements);
-            if (Math.random() > 0.5) {
-              // Special case - an update that doesn't change data
-              await pool.query(...updateStatements);
-            }
-          }
-
-          const deleteStatements: pgwire.Statement[] = ids.map((id) => {
-            return {
-              statement: `DELETE FROM test_data WHERE id = $1`,
-              params: [{ type: 'uuid', value: id }]
-            };
-          });
-          await pool.query(...deleteStatements);
-
-          await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
-        }
-      };
-
-      let compactController = new AbortController();
-
-      const bgCompact = async () => {
-        // Repeatedly compact, and check that the compact conditions hold
-        while (!compactController.signal.aborted) {
-          const delay = Math.random() * 50;
-          try {
-            await timers.setTimeout(delay, undefined, { signal: compactController.signal });
-          } catch (e) {
-            break;
-          }
-
-          const checkpoint = (await storage.getCheckpoint()).checkpoint;
-          if (f instanceof mongo_storage.storage.MongoBucketStorage) {
-            const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
-              .filter((row) => row._id.o <= checkpoint)
-              .map(mongo_storage.storage.mapOpEntry);
-            await storage.compact({ maxOpId: checkpoint });
-            const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
-              .filter((row) => row._id.o <= checkpoint)
-              .map(mongo_storage.storage.mapOpEntry);
-
-            test_utils.validateCompactedBucket(opsBefore, opsAfter);
-          } else if (f instanceof postgres_storage.PostgresBucketStorageFactory) {
-            const { db } = f;
-            const opsBefore = (
-              await db.sql`
-                SELECT
-                  *
-                FROM
-                  bucket_data
-                WHERE
-                  op_id <= ${{ type: 'int8', value: checkpoint }}
-                ORDER BY
-                  op_id ASC
-              `
-                .decoded(postgres_storage.models.BucketData)
-                .rows()
-            ).map(postgres_storage.utils.mapOpEntry);
-            await storage.compact({ maxOpId: checkpoint });
-            const opsAfter = (
-              await db.sql`
-                SELECT
-                  *
-                FROM
-                  bucket_data
-                WHERE
-                  op_id <= ${{ type: 'int8', value: checkpoint }}
-                ORDER BY
-                  op_id ASC
-              `
-                .decoded(postgres_storage.models.BucketData)
-                .rows()
-            ).map(postgres_storage.utils.mapOpEntry);
-
-            test_utils.validateCompactedBucket(opsBefore, opsAfter);
-          }
-        }
-      };
-
-      // Call the above loop multiple times concurrently
-      const promises = [1, 2, 3].map((i) => bg());
-      const compactPromise = testOptions.compact ? bgCompact() : null;
-      await Promise.all(promises);
-      compactController.abort();
-      await compactPromise;
-
-      // Wait for replication to finish
-      let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS });
-
-      if (f instanceof mongo_storage.storage.MongoBucketStorage) {
-        // Check that all inserts have been deleted again
-        const docs = await f.db.current_data.find().toArray();
-        const transformed = docs.map((doc) => {
-          return bson.deserialize(doc.data.buffer) as SqliteRow;
-        });
-        expect(transformed).toEqual([]);
-
-        // Check that each PUT has a REMOVE
-        const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
-
-        // All a single bucket in this test
-        const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op));
-        const reduced = test_utils.reduceBucket(bucket);
-        expect(reduced).toMatchObject([
-          {
-            op_id: '0',
-            op: 'CLEAR'
-          }
-          // Should contain no additional data
-        ]);
-      } else if (f instanceof postgres_storage.storage.PostgresBucketStorageFactory) {
-        const { db } = f;
-        // Check that all inserts have been deleted again
-        const docs = await db.sql`
-          SELECT
-            *
-          FROM
-            current_data
-        `
-          .decoded(postgres_storage.models.CurrentData)
-          .rows();
-        const transformed = docs.map((doc) => {
-          return bson.deserialize(doc.data) as SqliteRow;
-        });
-        expect(transformed).toEqual([]);
-
-        // Check that each PUT has a REMOVE
-        const ops = await db.sql`
-          SELECT
-            *
-          FROM
-            bucket_data
-          ORDER BY
-            op_id ASC
-        `
-          .decoded(postgres_storage.models.BucketData)
-          .rows();
-
-        // All a single bucket in this test
-        const bucket = ops.map((op) => postgres_storage.utils.mapOpEntry(op));
-        const reduced = test_utils.reduceBucket(bucket);
-        expect(reduced).toMatchObject([
-          {
-            op_id: '0',
-            op: 'CLEAR'
-          }
-          // Should contain no additional data
-        ]);
-      }
-    }
-
-    abortController.abort();
-    await streamPromise;
-  }
-
-  // Test repeatedly performing initial replication.
-  //
-  // If the first LSN does not correctly match with the first replication transaction,
-  // we may miss some updates.
-  test('repeated initial replication', { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }, async () => {
-    const pool = await connectPgPool();
-    await clearTestDb(pool);
-    await using f = await factory();
-
-    const syncRuleContent = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test_data"
-`;
-
-    const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion }));
-    const storage = f.getInstance(syncRules);
-
-    // 1. Setup some base data that will be replicated in initial replication
-    await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-    let statements: pgwire.Statement[] = [];
-
-    const n = Math.floor(Math.random() * 200);
-    for (let i = 0; i < n; i++) {
-      statements.push({
-        statement: `INSERT INTO test_data(description) VALUES('test_init')`
-      });
-    }
-    await pool.query(...statements);
-
-    const start = Date.now();
-    let i = 0;
-
-    while (Date.now() - start < TEST_DURATION_MS) {
-      // 2. Each iteration starts with a clean slate
-      await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`);
-      i += 1;
-
-      const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-      const replicationConnection = await connections.replicationConnection();
-
-      abortController = new AbortController();
-      const options: WalStreamOptions = {
-        abort_signal: abortController.signal,
-        connections,
-        storage: storage,
-        metrics: METRICS_HELPER.metricsEngine
-      };
-      walStream = new WalStream(options);
-
-      await storage.clear();
-
-      // 3. Start initial replication, then streaming, but don't wait for any of this
-      let initialReplicationDone = false;
-      streamPromise = (async () => {
-        await walStream.initReplication(replicationConnection);
-        initialReplicationDone = true;
-        await walStream.streamChanges(replicationConnection);
-      })()
-        .catch((e) => {
-          initialReplicationDone = true;
-          throw e;
-        })
-        .then((v) => {
-          return v;
-        });
-
-      // 4. While initial replication is still running, write more changes
-      while (!initialReplicationDone) {
-        let statements: pgwire.Statement[] = [];
-        const n = Math.floor(Math.random() * 10) + 1;
-        for (let i = 0; i < n; i++) {
-          const description = `test${i}`;
-          statements.push({
-            statement: `INSERT INTO test_data(description) VALUES('test1') returning id as test_id`,
-            params: [{ type: 'varchar', value: description }]
-          });
-        }
-        const results = await pool.query(...statements);
-        const ids = results.results.map((sub) => {
-          return sub.rows[0].decodeWithoutCustomTypes(0) as string;
-        });
-        await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
-        const deleteStatements: pgwire.Statement[] = ids.map((id) => {
-          return {
-            statement: `DELETE FROM test_data WHERE id = $1`,
-            params: [{ type: 'uuid', value: id }]
-          };
-        });
-        await pool.query(...deleteStatements);
-        await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
-      }
-
-      // 5. Once initial replication is done, wait for the streaming changes to complete syncing.
-      // getClientCheckpoint() effectively waits for the above replication to complete
-      // Race with streamingPromise to catch replication errors here.
-      let checkpoint = await Promise.race([
-        getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }),
-        streamPromise
-      ]);
-      if (checkpoint == null) {
-        // This indicates an issue with the test setup - streamingPromise completed instead
-        // of getClientCheckpoint()
-        throw new Error('Test failure - streamingPromise completed');
-      }
-
-      abortController.abort();
-      await streamPromise;
-      await connections.end();
-    }
-  });
-}

package/test/src/storage_combination.test.ts

@@ -1,35 +0,0 @@
-import * as postgres_storage from '@powersync/service-module-postgres-storage';
-import { describe, expect, test } from 'vitest';
-import { env } from './env.js';
-import { WalStreamTestContext } from './wal_stream_utils.js';
-
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)('replication storage combination - postgres', function () {
-  test('should allow the same Postgres cluster to be used for data and storage', async () => {
-    // Use the same cluster for the storage as the data source
-    await using context = await WalStreamTestContext.open(
-      postgres_storage.test_utils.postgresTestStorageFactoryGenerator({
-        url: env.PG_TEST_URL
-      }),
-      { doNotClear: false }
-    );
-
-    await context.updateSyncRules(/* yaml */
-    ` bucket_definitions:
-        global:
-          data:
-            - SELECT * FROM "test_data" `);
-
-    const { pool, connectionManager } = context;
-
-    const sourceVersion = await connectionManager.getServerVersion();
-
-    await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-    if (sourceVersion!.compareMain('14.0.0') < 0) {
-      await expect(context.replicateSnapshot()).rejects.toThrow();
-    } else {
-      // Should resolve
-      await context.replicateSnapshot();
-    }
-  });
-});

package/test/src/types/registry.test.ts

@@ -1,149 +0,0 @@
-import { describe, expect, test, beforeEach } from 'vitest';
-import { CustomTypeRegistry } from '@module/types/registry.js';
-import { CHAR_CODE_COMMA, PgTypeOid } from '@powersync/service-jpgwire';
-import {
-  applyValueContext,
-  CompatibilityContext,
-  CompatibilityEdition,
-  toSyncRulesValue
-} from '@powersync/service-sync-rules';
-
-describe('custom type registry', () => {
-  let registry: CustomTypeRegistry;
-
-  beforeEach(() => {
-    registry = new CustomTypeRegistry();
-  });
-
-  function checkResult(raw: string, type: number, old: any, fixed: any) {
-    const input = registry.decodeDatabaseValue(raw, type);
-    const syncRulesValue = toSyncRulesValue(input);
-
-    expect(applyValueContext(syncRulesValue, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toStrictEqual(old);
-    expect(
-      applyValueContext(syncRulesValue, new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }))
-    ).toStrictEqual(fixed);
-  }
-
-  test('domain types', () => {
-    registry.setDomainType(1337, PgTypeOid.INT4); // create domain wrapping integer
-    checkResult('12', 1337, '12', 12n); // Should be raw text value without fix, parsed as inner type if enabled
-  });
-
-  test('array of domain types', () => {
-    registry.setDomainType(1337, PgTypeOid.INT4);
-    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
-
-    checkResult('{1,2,3}', 1338, '{1,2,3}', '[1,2,3]');
-  });
-
-  test('nested array through domain type', () => {
-    registry.setDomainType(1337, PgTypeOid.INT4);
-    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
-    registry.setDomainType(1339, 1338);
-
-    checkResult('{1,2,3}', 1339, '{1,2,3}', '[1,2,3]');
-
-    registry.set(1400, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1339, sqliteType: () => 'text' });
-    checkResult('{{1,2,3}}', 1400, '{{1,2,3}}', '[[1,2,3]]');
-  });
-
-  test('structure', () => {
-    // create type c1 AS (a bool, b integer, c text[]);
-    registry.set(1337, {
-      type: 'composite',
-      sqliteType: () => 'text',
-      members: [
-        { name: 'a', typeId: PgTypeOid.BOOL },
-        { name: 'b', typeId: PgTypeOid.INT4 },
-        { name: 'c', typeId: 1009 } // text array
-      ]
-    });
-
-    // SELECT (TRUE, 123, ARRAY['foo', 'bar'])::c1;
-    checkResult('(t,123,"{foo,bar}")', 1337, '(t,123,"{foo,bar}")', '{"a":1,"b":123,"c":["foo","bar"]}');
-  });
-
-  test('array of structure', () => {
-    // create type c1 AS (a bool, b integer, c text[]);
-    registry.set(1337, {
-      type: 'composite',
-      sqliteType: () => 'text',
-      members: [
-        { name: 'a', typeId: PgTypeOid.BOOL },
-        { name: 'b', typeId: PgTypeOid.INT4 },
-        { name: 'c', typeId: 1009 } // text array
-      ]
-    });
-    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
-
-    // SELECT ARRAY[(TRUE, 123, ARRAY['foo', 'bar']),(FALSE, NULL, ARRAY[]::text[])]::c1[];
-    checkResult(
-      '{"(t,123,\\"{foo,bar}\\")","(f,,{})"}',
-      1338,
-      '{"(t,123,\\"{foo,bar}\\")","(f,,{})"}',
-      '[{"a":1,"b":123,"c":["foo","bar"]},{"a":0,"b":null,"c":[]}]'
-    );
-  });
-
-  test('domain type of structure', () => {
-    registry.set(1337, {
-      type: 'composite',
-      sqliteType: () => 'text',
-      members: [
-        { name: 'a', typeId: PgTypeOid.BOOL },
-        { name: 'b', typeId: PgTypeOid.INT4 }
-      ]
-    });
-    registry.setDomainType(1338, 1337);
-
-    checkResult('(t,123)', 1337, '(t,123)', '{"a":1,"b":123}');
-  });
-
-  test('structure of another structure', () => {
-    // CREATE TYPE c2 AS (a BOOLEAN, b INTEGER);
-    registry.set(1337, {
-      type: 'composite',
-      sqliteType: () => 'text',
-      members: [
-        { name: 'a', typeId: PgTypeOid.BOOL },
-        { name: 'b', typeId: PgTypeOid.INT4 }
-      ]
-    });
-    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
-    // CREATE TYPE c3 (c c2[]);
-    registry.set(1339, {
-      type: 'composite',
-      sqliteType: () => 'text',
-      members: [{ name: 'c', typeId: 1338 }]
-    });
-
-    // SELECT ROW(ARRAY[(FALSE,2)]::c2[])::c3;
-    checkResult('("{""(f,2)""}")', 1339, '("{""(f,2)""}")', '{"c":[{"a":0,"b":2}]}');
-  });
-
-  test('range', () => {
-    registry.set(1337, {
-      type: 'range',
-      sqliteType: () => 'text',
-      innerId: PgTypeOid.INT2
-    });
-
-    checkResult('[1,2]', 1337, '[1,2]', '{"lower":1,"upper":2,"lower_exclusive":0,"upper_exclusive":0}');
-  });
-
-  test('multirange', () => {
-    registry.set(1337, {
-      type: 'multirange',
-      sqliteType: () => 'text',
-      innerId: PgTypeOid.INT2
-    });
-
-    checkResult(
-      '{[1,2),[3,4)}',
-      1337,
-      '{[1,2),[3,4)}',
-      '[{"lower":1,"upper":2,"lower_exclusive":0,"upper_exclusive":1},{"lower":3,"upper":4,"lower_exclusive":0,"upper_exclusive":1}]'
-    );
-  });
-});