@powersync/service-module-postgres 0.19.3 → 0.19.4

package/src/utils/populate_test_data_worker.ts

@@ -1,53 +0,0 @@
-import * as crypto from 'crypto';
-import { isMainThread, parentPort, workerData } from 'node:worker_threads';
-
-import * as pgwire from '@powersync/service-jpgwire';
-import type { PopulateDataOptions } from './populate_test_data.js';
-
-// This util is actually for tests only, but we need it compiled to JS for the service to work, so it's placed in the service.
-
-if (isMainThread || parentPort == null) {
-  // Must not be imported - only expected to run in a worker
-  throw new Error('Do not import this file');
-} else {
-  try {
-    const options = workerData as PopulateDataOptions;
-    if (options == null) {
-      throw new Error('loaded worker without options');
-    }
-
-    const result = await populateDataInner(options);
-    parentPort.postMessage(result);
-    process.exit(0);
-  } catch (e) {
-    // This is a bug, not a connection issue
-    console.error(e);
-    // Only closes the Worker thread
-    process.exit(2);
-  }
-}
-
-async function populateDataInner(options: PopulateDataOptions) {
-  // Dedicated connection so we can release the memory easily
-  const initialDb = await pgwire.connectPgWire(options.connection, {
-    type: 'standard',
-    applicationName: 'powersync-tests'
-  });
-  const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
-  let operation_count = 0;
-  for (let i = 0; i < options.num_transactions; i++) {
-    const prefix = `test${i}K`;
-
-    await initialDb.query({
-      statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
-      params: [
-        { type: 'varchar', value: prefix },
-        { type: 'varchar', value: largeDescription },
-        { type: 'int4', value: options.per_transaction }
-      ]
-    });
-    operation_count += options.per_transaction;
-  }
-  await initialDb.end();
-  return operation_count;
-}

package/src/utils/postgres_version.ts

@@ -1,8 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import semver, { type SemVer } from 'semver';
-
-export async function getServerVersion(db: pgwire.PgClient): Promise<SemVer | null> {
-  const result = await db.query(`SHOW server_version;`);
-  // The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
-  return semver.coerce(result.rows[0].decodeWithoutCustomTypes(0).split(' ')[0]);
-}

package/test/src/checkpoints.test.ts

@@ -1,86 +0,0 @@
-import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js';
-import { checkpointUserId, createWriteCheckpoint } from '@powersync/service-core';
-import { describe, test } from 'vitest';
-import { describeWithStorage, StorageVersionTestContext } from './util.js';
-import { WalStreamTestContext } from './wal_stream_utils.js';
-
-import timers from 'node:timers/promises';
-
-const BASIC_SYNC_RULES = `bucket_definitions:
-  global:
-    data:
-      - SELECT id, description, other FROM "test_data"`;
-
-describe('checkpoint tests', () => {
-  describeWithStorage({}, checkpointTests);
-});
-
-const checkpointTests = ({ factory, storageVersion }: StorageVersionTestContext) => {
-  test('write checkpoints', { timeout: 50_000 }, async () => {
-    await using context = await WalStreamTestContext.open(factory, { storageVersion });
-
-    await context.updateSyncRules(BASIC_SYNC_RULES);
-    const { pool } = context;
-    const api = new PostgresRouteAPIAdapter(pool);
-    const serverVersion = await context.connectionManager.getServerVersion();
-    if (serverVersion!.compareMain('14.0.0') < 0) {
-      // The test is not stable on Postgres 11 or 12. See the notes on
-      // PostgresRouteAPIAdapter.createReplicationHead() for details.
-      // Postgres 12 is already EOL, so not worth finding a fix - just skip the tests.
-      // Postgres 13 fares a little better, but even there the test is still unstable.
-      // Postgres 14+ appears to have no issue.
-      console.log('Skipping write checkpoint test on Postgres < 14.0.0');
-      return;
-    }
-
-    await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-    // Wait for a consistent checkpoint before we start.
-    await context.initializeReplication();
-
-    const storage = context.storage!;
-
-    const controller = new AbortController();
-    try {
-      const stream = storage.watchCheckpointChanges({
-        user_id: checkpointUserId('test_user', 'test_client'),
-        signal: controller.signal
-      });
-
-      let lastWriteCheckpoint: bigint | null = null;
-
-      (async () => {
-        try {
-          for await (const cp of stream) {
-            lastWriteCheckpoint = cp.writeCheckpoint;
-          }
-        } catch (e) {
-          if (e.name != 'AbortError') {
-            throw e;
-          }
-        }
-      })();
-
-      for (let i = 0; i < 10; i++) {
-        const cp = await createWriteCheckpoint({
-          userId: 'test_user',
-          clientId: 'test_client',
-          api,
-          storage: context.factory
-        });
-
-        const start = Date.now();
-        while (lastWriteCheckpoint == null || lastWriteCheckpoint < BigInt(cp.writeCheckpoint)) {
-          if (Date.now() - start > 3_000) {
-            throw new Error(
-              `Timeout while waiting for checkpoint. last: ${lastWriteCheckpoint}, waiting for: ${cp.writeCheckpoint}`
-            );
-          }
-          await timers.setTimeout(5, undefined, { signal: controller.signal });
-        }
-      }
-    } finally {
-      controller.abort();
-    }
-  });
-};

package/test/src/chunked_snapshots.test.ts

@@ -1,161 +0,0 @@
-import { reduceBucket } from '@powersync/service-core';
-import { METRICS_HELPER } from '@powersync/service-core-tests';
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as crypto from 'node:crypto';
-import * as timers from 'timers/promises';
-import { describe, expect, test } from 'vitest';
-import { describeWithStorage, StorageVersionTestContext } from './util.js';
-import { WalStreamTestContext } from './wal_stream_utils.js';
-
-describe('chunked snapshots', () => {
-  describeWithStorage({ timeout: 120_000 }, defineBatchTests);
-});
-
-function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
-  const openContext = (options?: Parameters<typeof WalStreamTestContext.open>[1]) => {
-    return WalStreamTestContext.open(factory, { ...options, storageVersion });
-  };
-
-  // We need to test every supported type, since chunking could be quite sensitive to
-  // how each specific type is handled.
-  test('chunked snapshot edge case (int2)', async () => {
-    await testChunkedSnapshot({
-      idType: 'int2',
-      genId: 'i',
-      lastId: '2000',
-      moveTo: '0',
-      moveToJs: 0
-    });
-  });
-
-  test('chunked snapshot edge case (int4)', async () => {
-    await testChunkedSnapshot({
-      idType: 'int4',
-      genId: 'i',
-      lastId: '2000',
-      moveTo: '0',
-      moveToJs: 0
-    });
-  });
-
-  test('chunked snapshot edge case (int8)', async () => {
-    await testChunkedSnapshot({
-      idType: 'int8',
-      genId: 'i',
-      lastId: '2000',
-      moveTo: '0',
-      moveToJs: 0
-    });
-  });
-
-  test('chunked snapshot edge case (text)', async () => {
-    await testChunkedSnapshot({
-      idType: 'text',
-      genId: `to_char(i, 'fm0000')`,
-      lastId: `'2000'`,
-      moveTo: `'0000'`,
-      moveToJs: '0000'
-    });
-  });
-
-  test('chunked snapshot edge case (varchar)', async () => {
-    await testChunkedSnapshot({
-      idType: 'varchar',
-      genId: `to_char(i, 'fm0000')`,
-      lastId: `'2000'`,
-      moveTo: `'0000'`,
-      moveToJs: '0000'
-    });
-  });
-
-  test('chunked snapshot edge case (uuid)', async () => {
-    await testChunkedSnapshot({
-      idType: 'uuid',
-      // Generate a uuid by using the first part of a uuid and appending a 4-digit number.
-      genId: `('00000000-0000-4000-8000-00000000' || to_char(i, 'fm0000')) :: uuid`,
-      lastId: `'00000000-0000-4000-8000-000000002000'`,
-      moveTo: `'00000000-0000-4000-8000-000000000000'`,
-      moveToJs: '00000000-0000-4000-8000-000000000000'
-    });
-  });
-
-  async function testChunkedSnapshot(options: {
-    idType: string;
-    genId: string;
-    lastId: string;
-    moveTo: string;
-    moveToJs: SqliteJsonValue;
-  }) {
-    // 1. Start with 2k rows, one row with id = 2000, and a large TOAST value in another column.
-    // 2. Replicate one batch of rows (id < 2000).
-    // 3. `UPDATE table SET id = 0 WHERE id = 2000`
-    // 4. Replicate the rest of the table.
-    // 5. Logical replication picks up the UPDATE above, but it is missing the TOAST column.
-    // 6. We end up with a row that has a missing TOAST column.
-
-    await using context = await openContext({
-      // We need to use a smaller chunk size here, so that we can run a query in between chunks
-      walStreamOptions: { snapshotChunkLength: 100 }
-    });
-
-    await context.updateSyncRules(`bucket_definitions:
-  global:
-    data:
-      - SELECT * FROM test_data`);
-    const { pool } = context;
-
-    await pool.query(`CREATE TABLE test_data(id ${options.idType} primary key, description text)`);
-
-    // 1. Start with 2k rows, one row with id = 2000...
-    await pool.query({
-      statement: `INSERT INTO test_data(id, description) SELECT ${options.genId}, 'foo' FROM generate_series(1, 2000) i`
-    });
-
-    // ...and a large TOAST value in another column.
-    // Toast value, must be > 8kb after compression
-    const largeDescription = crypto.randomBytes(20_000).toString('hex');
-    await pool.query({
-      statement: `UPDATE test_data SET description = $1 WHERE id = ${options.lastId} :: ${options.idType}`,
-      params: [{ type: 'varchar', value: largeDescription }]
-    });
-
-    // 2. Replicate one batch of rows (id < 100).
-    // Our "stopping point" here is not quite deterministic.
-    const p = context.replicateSnapshot();
-
-    const stopAfter = 100;
-    const startRowCount = (await METRICS_HELPER.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-
-    while (true) {
-      const count =
-        ((await METRICS_HELPER.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0) - startRowCount;
-
-      if (count >= stopAfter) {
-        break;
-      }
-      await timers.setTimeout(1);
-    }
-
-    // 3. `UPDATE table SET id = 0 WHERE id = 2000`
-    const rs = await pool.query(
-      `UPDATE test_data SET id = ${options.moveTo} WHERE id = ${options.lastId} RETURNING id`
-    );
-    expect(rs.rows.length).toEqual(1);
-
-    // 4. Replicate the rest of the table.
-    await p;
-
-    // 5. Logical replication picks up the UPDATE above, but it is missing the TOAST column.
-    // Note: logical replication now runs concurrently with the snapshot.
-    // TODO: re-check the test logic here.
-
-    // 6. If all went well, the "resnapshot" process would take care of this.
-    const data = await context.getBucketData('global[]', undefined, {});
-    const reduced = reduceBucket(data);
-
-    const movedRow = reduced.find((row) => row.object_id == String(options.moveToJs));
-    expect(movedRow?.data).toEqual(JSON.stringify({ id: options.moveToJs, description: largeDescription }));
-
-    expect(reduced.length).toEqual(2001);
-  }
-}

package/test/src/env.ts

@@ -1,11 +0,0 @@
-import { utils } from '@powersync/lib-services-framework';
-
-export const env = utils.collectEnvironmentVariables({
-  PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
-  PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'),
-  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
-  CI: utils.type.boolean.default('false'),
-  SLOW_TESTS: utils.type.boolean.default('false'),
-  TEST_MONGO_STORAGE: utils.type.boolean.default('true'),
-  TEST_POSTGRES_STORAGE: utils.type.boolean.default('true')
-});

package/test/src/large_batch.test.ts

@@ -1,241 +0,0 @@
-import { describe, expect, test } from 'vitest';
-import { populateData } from '../../dist/utils/populate_test_data.js';
-import { env } from './env.js';
-import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
-import { WalStreamTestContext } from './wal_stream_utils.js';
-import { CURRENT_STORAGE_VERSION } from '@powersync/service-core';
-
-describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
-  describeWithStorage(
-    {
-      timeout: 240_000,
-      // These tests are slow, so only test the current storage version
-      storageVersions: [CURRENT_STORAGE_VERSION]
-    },
-    defineBatchTests
-  );
-});
-
-const BASIC_SYNC_RULES = `bucket_definitions:
-  global:
-    data:
-      - SELECT id, description, other FROM "test_data"`;
-
-function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
-  const openContext = (options?: Parameters<typeof WalStreamTestContext.open>[1]) => {
-    return WalStreamTestContext.open(factory, { ...options, storageVersion });
-  };
-
-  test('update large record', async () => {
-    await using context = await openContext();
-    // This test generates a large transaction in MongoDB, despite the replicated data
-    // not being that large.
-    // If we don't limit transaction size, we could run into this error:
-    // > -31800: transaction is too large and will not fit in the storage engine cache
-    await context.updateSyncRules(BASIC_SYNC_RULES);
-    const { pool } = context;
-
-    await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-    await context.replicateSnapshot();
-
-    let operation_count = await populateData({
-      num_transactions: 1,
-      per_transaction: 80,
-      size: 4_000_000,
-      connection: TEST_CONNECTION_OPTIONS
-    });
-
-    const start = Date.now();
-
-    const checksum = await context.getChecksums(['global[]'], { timeout: 100_000 });
-    const duration = Date.now() - start;
-    const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-    expect(checksum.get('global[]')!.count).toEqual(operation_count);
-    const perSecond = Math.round((operation_count / duration) * 1000);
-    console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
-  });
-
-  test('initial replication performance', async () => {
-    await using context = await openContext();
-    // Manual test to check initial replication performance and memory usage
-    await context.updateSyncRules(BASIC_SYNC_RULES);
-    const { pool } = context;
-
-    await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-    // Some stats (varies a lot):
-    // Old 'postgres' driver, using cursor(2)
-    // 15 ops in 19559ms 1 ops/s. 354MB RSS, 115MB heap, 137MB external
-    // 25 ops in 42984ms 1 ops/s. 377MB RSS, 129MB heap, 137MB external
-    // 35 ops in 41337ms 1 ops/s. 365MB RSS, 115MB heap, 137MB external
-
-    // streaming with pgwire
-    // 15 ops in 26423ms 1 ops/s. 379MB RSS, 128MB heap, 182MB external, 165MB ArrayBuffers
-    // 35 ops in 78897ms 0 ops/s. 539MB RSS, 52MB heap, 87MB external, 83MB ArrayBuffers
-
-    let operation_count = await populateData({
-      num_transactions: 1,
-      per_transaction: 35,
-      size: 14_000_000,
-      connection: TEST_CONNECTION_OPTIONS
-    });
-
-    global.gc?.();
-
-    // Note that we could already have high memory usage at this point
-    printMemoryUsage();
-
-    let interval = setInterval(() => {
-      printMemoryUsage();
-    }, 2000);
-    try {
-      const start = Date.now();
-
-      await context.replicateSnapshot();
-
-      const checksum = await context.getChecksums(['global[]'], { timeout: 100_000 });
-      const duration = Date.now() - start;
-      expect(checksum.get('global[]')!.count).toEqual(operation_count);
-      const perSecond = Math.round((operation_count / duration) * 1000);
-      console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s.`);
-      printMemoryUsage();
-    } finally {
-      clearInterval(interval);
-    }
-  });
-
-  test('large number of operations', async () => {
-    await using context = await openContext();
-    // This just tests performance of a large number of operations inside a transaction.
-    await context.updateSyncRules(BASIC_SYNC_RULES);
-    const { pool } = context;
-
-    await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-    await context.replicateSnapshot();
-
-    const numTransactions = 20;
-    const perTransaction = 1500;
-    let operationCount = 0;
-
-    const description = 'description';
-
-    for (let i = 0; i < numTransactions; i++) {
-      const prefix = `test${i}K`;
-
-      await pool.query(
-        {
-          statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2 || i, 'foo' FROM generate_series(1, $3) i`,
-          params: [
-            { type: 'varchar', value: prefix },
-            { type: 'varchar', value: description },
-            { type: 'int4', value: perTransaction }
-          ]
-        },
-        {
-          statement: `UPDATE test_data SET other = other || '#' WHERE id LIKE $1 || '%'`,
-          params: [{ type: 'varchar', value: prefix }]
-        }
-      );
-      operationCount += perTransaction * 2;
-    }
-
-    const start = Date.now();
-
-    const checksum = await context.getChecksums(['global[]']);
-    const duration = Date.now() - start;
-    const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-    expect(checksum.get('global[]')!.count).toEqual(operationCount);
-    const perSecond = Math.round((operationCount / duration) * 1000);
-    // This number depends on the test machine, so we keep the test significantly
-    // lower than expected numbers.
-    expect(perSecond).toBeGreaterThan(1000);
-    console.log(`${operationCount} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
-
-    // Truncating is fast (~10k ops/second).
-    // We'd need a really large set of data to actually run into limits when truncating,
-    // but we just test with the data we have here.
-    const truncateStart = Date.now();
-    await pool.query(`TRUNCATE test_data`);
-
-    const checksum2 = await context.getChecksums(['global[]'], { timeout: 20_000 });
-    const truncateDuration = Date.now() - truncateStart;
-    const truncateCount = checksum2.get('global[]')!.count - checksum.get('global[]')!.count;
-    expect(truncateCount).toEqual(numTransactions * perTransaction);
-    const truncatePerSecond = Math.round((truncateCount / truncateDuration) * 1000);
-    console.log(`Truncated ${truncateCount} ops in ${truncateDuration}ms ${truncatePerSecond} ops/s. ${used}MB heap`);
-  });
-
-  test('large number of bucket_data docs', async () => {
-    // This tests that we don't run into this error:
-    //   MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
-    // The test is quite sensitive to internals, since we need to
-    // generate an internal batch that is just below 16MB.
-    //
-    // For the test to work, we need a:
-    // 1. Large number of documents in the batch.
-    // 2. More bucket_data documents than current_data documents,
-    //    otherwise other batch limiting thresholds are hit.
-    // 3. A large document to make sure we get to just below the 16MB
-    //    limit.
-    // 4. Another document to make sure the internal batching overflows
-    //    to a second batch.
-
-    await using context = await openContext();
-    await context.updateSyncRules(`bucket_definitions:
-  global:
-    data:
-      # Sync 4x so we get more bucket_data documents
-      - SELECT * FROM test_data
-      - SELECT * FROM test_data
-      - SELECT * FROM test_data
-      - SELECT * FROM test_data
-      `);
-    const { pool } = context;
-
-    await pool.query(`CREATE TABLE test_data(id serial primary key, description text)`);
-
-    const numDocs = 499;
-    let description = '';
-    while (description.length < 2650) {
-      description += '.';
-    }
-
-    await pool.query({
-      statement: `INSERT INTO test_data(description) SELECT $2 FROM generate_series(1, $1) i`,
-      params: [
-        { type: 'int4', value: numDocs },
-        { type: 'varchar', value: description }
-      ]
-    });
-
-    let largeDescription = '';
-
-    while (largeDescription.length < 2_768_000) {
-      largeDescription += '.';
-    }
-    await pool.query({
-      statement: 'INSERT INTO test_data(description) VALUES($1)',
-      params: [{ type: 'varchar', value: largeDescription }]
-    });
-    await pool.query({
-      statement: 'INSERT INTO test_data(description) VALUES($1)',
-      params: [{ type: 'varchar', value: 'testingthis' }]
-    });
-    await context.replicateSnapshot();
-
-    const checksum = await context.getChecksums(['global[]'], { timeout: 50_000 });
-    expect(checksum.get('global[]')!.count).toEqual((numDocs + 2) * 4);
-  });
-
-  function printMemoryUsage() {
-    const memoryUsage = process.memoryUsage();
-
-    const rss = Math.round(memoryUsage.rss / 1024 / 1024);
-    const heap = Math.round(memoryUsage.heapUsed / 1024 / 1024);
-    const external = Math.round(memoryUsage.external / 1024 / 1024);
-    const arrayBuffers = Math.round(memoryUsage.arrayBuffers / 1024 / 1024);
-    console.log(`${rss}MB RSS, ${heap}MB heap, ${external}MB external, ${arrayBuffers}MB ArrayBuffers`);
-  }
-}