@powersync/service-core 0.8.8 → 0.10.0

package/src/util/pgwire_utils.ts

@@ -1,139 +0,0 @@
-// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
-
-import * as bson from 'bson';
-import * as uuid from 'uuid';
-import * as pgwire from '@powersync/service-jpgwire';
-import { SqliteJsonValue, SqliteRow, ToastableSqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
-
-import * as replication from '../replication/replication-index.js';
-import { logger } from '@powersync/lib-services-framework';
-
-/**
- * pgwire message -> SQLite row.
- * @param message
- */
-export function constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteRow {
-  const rawData = (message as any).afterRaw;
-
-  const record = pgwire.decodeTuple(message.relation, rawData);
-  return toSyncRulesRow(record);
-}
-
-export function hasToastedValues(row: ToastableSqliteRow) {
-  for (let key in row) {
-    if (typeof row[key] == 'undefined') {
-      return true;
-    }
-  }
-  return false;
-}
-
-export function isCompleteRow(row: ToastableSqliteRow): row is SqliteRow {
-  return !hasToastedValues(row);
-}
-
-/**
- * pgwire message -> SQLite row.
- * @param message
- */
-export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteRow | undefined {
-  const rawData = (message as any).beforeRaw;
-  if (rawData == null) {
-    return undefined;
-  }
-  const record = pgwire.decodeTuple(message.relation, rawData);
-  return toSyncRulesRow(record);
-}
-
-function getRawReplicaIdentity(
-  tuple: ToastableSqliteRow,
-  columns: replication.ReplicationColumn[]
-): Record<string, any> {
-  let result: Record<string, any> = {};
-  for (let column of columns) {
-    const name = column.name;
-    result[name] = tuple[name];
-  }
-  return result;
-}
-const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
-
-export function getUuidReplicaIdentityString(
-  tuple: ToastableSqliteRow,
-  columns: replication.ReplicationColumn[]
-): string {
-  const rawIdentity = getRawReplicaIdentity(tuple, columns);
-
-  return uuidForRow(rawIdentity);
-}
-
-export function uuidForRow(row: SqliteRow): string {
-  // Important: This must not change, since it will affect how ids are generated.
-  // Use BSON so that it's a well-defined format without encoding ambiguities.
-  const repr = bson.serialize(row);
-  return uuid.v5(repr, ID_NAMESPACE);
-}
-
-export function getUuidReplicaIdentityBson(
-  tuple: ToastableSqliteRow,
-  columns: replication.ReplicationColumn[]
-): bson.UUID {
-  if (columns.length == 0) {
-    // REPLICA IDENTITY NOTHING - generate random id
-    return new bson.UUID(uuid.v4());
-  }
-  const rawIdentity = getRawReplicaIdentity(tuple, columns);
-
-  return uuidForRowBson(rawIdentity);
-}
-
-export function uuidForRowBson(row: SqliteRow): bson.UUID {
-  // Important: This must not change, since it will affect how ids are generated.
-  // Use BSON so that it's a well-defined format without encoding ambiguities.
-  const repr = bson.serialize(row);
-  const buffer = Buffer.alloc(16);
-  return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
-}
-
-export function escapeIdentifier(identifier: string) {
-  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
-}
-
-export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam {
-  if (arg == null) {
-    return { type: 'varchar', value: null };
-  } else if (typeof arg == 'string') {
-    return { type: 'varchar', value: arg };
-  } else if (typeof arg == 'number') {
-    if (Number.isInteger(arg)) {
-      return { type: 'int8', value: arg };
-    } else {
-      return { type: 'float8', value: arg };
-    }
-  } else if (typeof arg == 'boolean') {
-    return { type: 'bool', value: arg };
-  } else if (typeof arg == 'bigint') {
-    return { type: 'int8', value: arg };
-  } else {
-    throw new Error(`Unsupported query parameter: ${typeof arg}`);
-  }
-}
-
-export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise<pgwire.PgResult>;
-export async function retriedQuery(db: pgwire.PgClient, query: string): Promise<pgwire.PgResult>;
-
-/**
- * Retry a simple query - up to 2 attempts total.
- */
-export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) {
-  for (let tries = 2; ; tries--) {
-    try {
-      return await db.query(...args);
-    } catch (e) {
-      if (tries == 1) {
-        throw e;
-      }
-      logger.warn('Query error, retrying', e);
-    }
-  }
-}

package/src/util/populate_test_data.ts

@@ -1,78 +0,0 @@
-import * as crypto from 'crypto';
-import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
-
-import { connectPgWire } from '@powersync/service-jpgwire';
-import { NormalizedPostgresConnection } from '@powersync/service-types';
-
-// This util is actually for tests only, but we need it compiled to JS for the service to work, so it's placed in the service.
-
-export interface PopulateDataOptions {
-  connection: NormalizedPostgresConnection;
-  num_transactions: number;
-  per_transaction: number;
-  size: number;
-}
-
-if (isMainThread || parentPort == null) {
-  // Not a worker - ignore
-} else {
-  try {
-    const options = workerData as PopulateDataOptions;
-
-    const result = await populateDataInner(options);
-    parentPort.postMessage(result);
-    process.exit(0);
-  } catch (e) {
-    // This is a bug, not a connection issue
-    console.error(e);
-    // Only closes the Worker thread
-    process.exit(2);
-  }
-}
-
-async function populateDataInner(options: PopulateDataOptions) {
-  // Dedicated connection so we can release the memory easily
-  const initialDb = await connectPgWire(options.connection, { type: 'standard' });
-  const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
-  let operation_count = 0;
-  for (let i = 0; i < options.num_transactions; i++) {
-    const prefix = `test${i}K`;
-
-    await initialDb.query({
-      statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
-      params: [
-        { type: 'varchar', value: prefix },
-        { type: 'varchar', value: largeDescription },
-        { type: 'int4', value: options.per_transaction }
-      ]
-    });
-    operation_count += options.per_transaction;
-  }
-  await initialDb.end();
-  return operation_count;
-}
-
-export async function populateData(options: PopulateDataOptions) {
-  const WORKER_TIMEOUT = 30_000;
-
-  const worker = new Worker(new URL('./populate_test_data.js', import.meta.url), {
-    workerData: options
-  });
-  const timeout = setTimeout(() => {
-    // Exits with code 1 below
-    worker.terminate();
-  }, WORKER_TIMEOUT);
-  try {
-    return await new Promise<number>((resolve, reject) => {
-      worker.on('message', resolve);
-      worker.on('error', reject);
-      worker.on('exit', (code) => {
-        if (code !== 0) {
-          reject(new Error(`Populating data failed with exit code ${code}`));
-        }
-      });
-    });
-  } finally {
-    clearTimeout(timeout);
-  }
-}

package/test/src/__snapshots__/pg_test.test.ts.snap

@@ -1,256 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`pg data types > schema 1`] = `
-[
-  {
-    "name": "public",
-    "tables": [
-      {
-        "columns": [
-          {
-            "name": "id",
-            "pg_type": "int4",
-            "type": "integer",
-          },
-          {
-            "name": "text",
-            "pg_type": "text",
-            "type": "text",
-          },
-          {
-            "name": "uuid",
-            "pg_type": "uuid",
-            "type": "uuid",
-          },
-          {
-            "name": "varchar",
-            "pg_type": "varchar",
-            "type": "character varying(255)",
-          },
-          {
-            "name": "bool",
-            "pg_type": "bool",
-            "type": "boolean",
-          },
-          {
-            "name": "bytea",
-            "pg_type": "bytea",
-            "type": "bytea",
-          },
-          {
-            "name": "int2",
-            "pg_type": "int2",
-            "type": "smallint",
-          },
-          {
-            "name": "int4",
-            "pg_type": "int4",
-            "type": "integer",
-          },
-          {
-            "name": "int8",
-            "pg_type": "int8",
-            "type": "bigint",
-          },
-          {
-            "name": "float4",
-            "pg_type": "float4",
-            "type": "real",
-          },
-          {
-            "name": "float8",
-            "pg_type": "float8",
-            "type": "double precision",
-          },
-          {
-            "name": "numeric",
-            "pg_type": "numeric",
-            "type": "numeric",
-          },
-          {
-            "name": "json",
-            "pg_type": "json",
-            "type": "json",
-          },
-          {
-            "name": "jsonb",
-            "pg_type": "jsonb",
-            "type": "jsonb",
-          },
-          {
-            "name": "pg_lsn",
-            "pg_type": "pg_lsn",
-            "type": "pg_lsn",
-          },
-          {
-            "name": "date",
-            "pg_type": "date",
-            "type": "date",
-          },
-          {
-            "name": "time",
-            "pg_type": "time",
-            "type": "time without time zone",
-          },
-          {
-            "name": "timestamp",
-            "pg_type": "timestamp",
-            "type": "timestamp without time zone",
-          },
-          {
-            "name": "timestamptz",
-            "pg_type": "timestamptz",
-            "type": "timestamp with time zone",
-          },
-          {
-            "name": "interval",
-            "pg_type": "interval",
-            "type": "interval",
-          },
-          {
-            "name": "macaddr",
-            "pg_type": "macaddr",
-            "type": "macaddr",
-          },
-          {
-            "name": "inet",
-            "pg_type": "inet",
-            "type": "inet",
-          },
-          {
-            "name": "oid",
-            "pg_type": "oid",
-            "type": "oid",
-          },
-        ],
-        "name": "test_data",
-      },
-      {
-        "columns": [
-          {
-            "name": "id",
-            "pg_type": "int4",
-            "type": "integer",
-          },
-          {
-            "name": "text",
-            "pg_type": "text[]",
-            "type": "text[]",
-          },
-          {
-            "name": "uuid",
-            "pg_type": "uuid[]",
-            "type": "uuid[]",
-          },
-          {
-            "name": "varchar",
-            "pg_type": "varchar[]",
-            "type": "character varying(255)[]",
-          },
-          {
-            "name": "bool",
-            "pg_type": "bool[]",
-            "type": "boolean[]",
-          },
-          {
-            "name": "bytea",
-            "pg_type": "bytea[]",
-            "type": "bytea[]",
-          },
-          {
-            "name": "int2",
-            "pg_type": "int2[]",
-            "type": "smallint[]",
-          },
-          {
-            "name": "int4",
-            "pg_type": "int4[]",
-            "type": "integer[]",
-          },
-          {
-            "name": "int8",
-            "pg_type": "int8[]",
-            "type": "bigint[]",
-          },
-          {
-            "name": "float4",
-            "pg_type": "float4[]",
-            "type": "real[]",
-          },
-          {
-            "name": "float8",
-            "pg_type": "float8[]",
-            "type": "double precision[]",
-          },
-          {
-            "name": "numeric",
-            "pg_type": "numeric[]",
-            "type": "numeric[]",
-          },
-          {
-            "name": "json",
-            "pg_type": "json[]",
-            "type": "json[]",
-          },
-          {
-            "name": "jsonb",
-            "pg_type": "jsonb[]",
-            "type": "jsonb[]",
-          },
-          {
-            "name": "pg_lsn",
-            "pg_type": "pg_lsn[]",
-            "type": "pg_lsn[]",
-          },
-          {
-            "name": "date",
-            "pg_type": "date[]",
-            "type": "date[]",
-          },
-          {
-            "name": "time",
-            "pg_type": "time[]",
-            "type": "time without time zone[]",
-          },
-          {
-            "name": "timestamp",
-            "pg_type": "timestamp[]",
-            "type": "timestamp without time zone[]",
-          },
-          {
-            "name": "timestamptz",
-            "pg_type": "timestamptz[]",
-            "type": "timestamp with time zone[]",
-          },
-          {
-            "name": "interval",
-            "pg_type": "interval[]",
-            "type": "interval[]",
-          },
-          {
-            "name": "macaddr",
-            "pg_type": "macaddr[]",
-            "type": "macaddr[]",
-          },
-          {
-            "name": "inet",
-            "pg_type": "inet[]",
-            "type": "inet[]",
-          },
-          {
-            "name": "oid",
-            "pg_type": "oid[]",
-            "type": "oid[]",
-          },
-          {
-            "name": "multidimensional",
-            "pg_type": "text[]",
-            "type": "text[]",
-          },
-        ],
-        "name": "test_data_arrays",
-      },
-    ],
-  },
-]
-`;

package/test/src/large_batch.test.ts

@@ -1,194 +0,0 @@
-import { describe, expect, test } from 'vitest';
-import { env } from './env.js';
-import { MONGO_STORAGE_FACTORY, StorageFactory, TEST_CONNECTION_OPTIONS } from './util.js';
-import { walStreamTest } from './wal_stream_utils.js';
-import { populateData } from '../../dist/util/populate_test_data.js';
-
-describe('batch replication tests - mongodb', function () {
-  // These are slow but consistent tests.
-  // Not run on every test run, but we do run on CI, or when manually debugging issues.
-  if (env.CI || env.SLOW_TESTS) {
-    defineBatchTests(MONGO_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
-});
-
-const BASIC_SYNC_RULES = `bucket_definitions:
-  global:
-    data:
-      - SELECT id, description, other FROM "test_data"`;
-
-function defineBatchTests(factory: StorageFactory) {
-  test(
-    'update large record',
-    walStreamTest(factory, async (context) => {
-      // This test generates a large transaction in MongoDB, despite the replicated data
-      // not being that large.
-      // If we don't limit transaction size, we could run into this error:
-      // > -31800: transaction is too large and will not fit in the storage engine cache
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-      const { pool } = context;
-
-      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-      await context.replicateSnapshot();
-
-      let operation_count = await populateData({
-        num_transactions: 1,
-        per_transaction: 80,
-        size: 4_000_000,
-        connection: TEST_CONNECTION_OPTIONS
-      });
-
-      const start = Date.now();
-
-      context.startStreaming();
-
-      const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
-      const duration = Date.now() - start;
-      const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-      const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
-      expect(checksum.get('global[]')!.count).toEqual(operation_count);
-      const perSecond = Math.round((operation_count / duration) * 1000);
-      console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
-    }),
-    { timeout: 120_000 }
-  );
-
-  test(
-    'initial replication performance',
-    walStreamTest(factory, async (context) => {
-      // Manual test to check initial replication performance and memory usage
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-      const { pool } = context;
-
-      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-      // Some stats (varies a lot):
-      // Old 'postgres' driver, using cursor(2)
-      // 15 ops in 19559ms 1 ops/s. 354MB RSS, 115MB heap, 137MB external
-      // 25 ops in 42984ms 1 ops/s. 377MB RSS, 129MB heap, 137MB external
-      // 35 ops in 41337ms 1 ops/s. 365MB RSS, 115MB heap, 137MB external
-
-      // streaming with pgwire
-      // 15 ops in 26423ms 1 ops/s. 379MB RSS, 128MB heap, 182MB external, 165MB ArrayBuffers
-      // 35 ops in 78897ms 0 ops/s. 539MB RSS, 52MB heap, 87MB external, 83MB ArrayBuffers
-
-      let operation_count = await populateData({
-        num_transactions: 1,
-        per_transaction: 35,
-        size: 14_000_000,
-        connection: TEST_CONNECTION_OPTIONS
-      });
-
-      global.gc?.();
-
-      // Note that we could already have high memory usage at this point
-      printMemoryUsage();
-
-      let interval = setInterval(() => {
-        printMemoryUsage();
-      }, 2000);
-      try {
-        const start = Date.now();
-
-        await context.replicateSnapshot();
-        await context.storage!.autoActivate();
-        context.startStreaming();
-
-        const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
-        const duration = Date.now() - start;
-        const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
-        expect(checksum.get('global[]')!.count).toEqual(operation_count);
-        const perSecond = Math.round((operation_count / duration) * 1000);
-        console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s.`);
-        printMemoryUsage();
-      } finally {
-        clearInterval(interval);
-      }
-    }),
-    { timeout: 120_000 }
-  );
-
-  test(
-    'large number of operations',
-    walStreamTest(factory, async (context) => {
-      // This just tests performance of a large number of operations inside a transaction.
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-      const { pool } = context;
-
-      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-      await context.replicateSnapshot();
-
-      const numTransactions = 20;
-      const perTransaction = 1500;
-      let operationCount = 0;
-
-      const description = 'description';
-
-      for (let i = 0; i < numTransactions; i++) {
-        const prefix = `test${i}K`;
-
-        await pool.query(
-          {
-            statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2 || i, 'foo' FROM generate_series(1, $3) i`,
-            params: [
-              { type: 'varchar', value: prefix },
-              { type: 'varchar', value: description },
-              { type: 'int4', value: perTransaction }
-            ]
-          },
-          {
-            statement: `UPDATE test_data SET other = other || '#' WHERE id LIKE $1 || '%'`,
-            params: [{ type: 'varchar', value: prefix }]
-          }
-        );
-        operationCount += perTransaction * 2;
-      }
-
-      const start = Date.now();
-
-      context.startStreaming();
-
-      const checkpoint = await context.getCheckpoint({ timeout: 50_000 });
-      const duration = Date.now() - start;
-      const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-      const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
-      expect(checksum.get('global[]')!.count).toEqual(operationCount);
-      const perSecond = Math.round((operationCount / duration) * 1000);
-      // This number depends on the test machine, so we keep the test significantly
-      // lower than expected numbers.
-      expect(perSecond).toBeGreaterThan(1000);
-      console.log(`${operationCount} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
-
-      // Truncating is fast (~10k ops/second).
-      // We'd need a really large set of data to actually run into limits when truncating,
-      // but we just test with the data we have here.
-      const truncateStart = Date.now();
-      await pool.query(`TRUNCATE test_data`);
-
-      const checkpoint2 = await context.getCheckpoint({ timeout: 20_000 });
-      const truncateDuration = Date.now() - truncateStart;
-
-      const checksum2 = await context.storage!.getChecksums(checkpoint2, ['global[]']);
-      const truncateCount = checksum2.get('global[]')!.count - checksum.get('global[]')!.count;
-      expect(truncateCount).toEqual(numTransactions * perTransaction);
-      const truncatePerSecond = Math.round((truncateCount / truncateDuration) * 1000);
-      console.log(`Truncated ${truncateCount} ops in ${truncateDuration}ms ${truncatePerSecond} ops/s. ${used}MB heap`);
-    }),
-    { timeout: 90_000 }
-  );
-
-  function printMemoryUsage() {
-    const memoryUsage = process.memoryUsage();
-
-    const rss = Math.round(memoryUsage.rss / 1024 / 1024);
-    const heap = Math.round(memoryUsage.heapUsed / 1024 / 1024);
-    const external = Math.round(memoryUsage.external / 1024 / 1024);
-    const arrayBuffers = Math.round(memoryUsage.arrayBuffers / 1024 / 1024);
-    console.log(`${rss}MB RSS, ${heap}MB heap, ${external}MB external, ${arrayBuffers}MB ArrayBuffers`);
-  }
-}