@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/src/system/CorePowerSyncSystem.ts

@@ -1,64 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { LifeCycledSystem, container, logger } from '@powersync/lib-services-framework';
-
-import * as storage from '../storage/storage-index.js';
-import * as utils from '../util/util-index.js';
-
-export abstract class CorePowerSyncSystem extends LifeCycledSystem {
-  abstract storage: storage.BucketStorageFactory;
-  abstract pgwire_pool?: pgwire.PgClient;
-  closed: boolean;
-
-  protected stopHandlers: Set<() => void> = new Set();
-
-  constructor(public config: utils.ResolvedPowerSyncConfig) {
-    super();
-    this.closed = false;
-  }
-
-  get client_keystore() {
-    return this.config.client_keystore;
-  }
-
-  get dev_client_keystore() {
-    return this.config.dev_client_keystore;
-  }
-
-  /**
-   * Adds a termination handler which will call handlers registered via
-   * [addStopHandler].
-   * This should be called after the server is started and it's termination handler is added.
-   * This is so that the handler is run before the server's handler, allowing streams to be interrupted on exit
-   */
-  addTerminationHandler() {
-    container.terminationHandler.handleTerminationSignal(async () => {
-      // Close open streams, so that they don't block the server from closing.
-      // Note: This does not work well when streaming requests are queued. In that case, the server still doesn't
-      // close in the 30-second timeout.
-      this.closed = true;
-      logger.info(`Closing ${this.stopHandlers.size} streams`);
-      for (let handler of this.stopHandlers) {
-        handler();
-      }
-    });
-  }
-
-  addStopHandler(handler: () => void): () => void {
-    if (this.closed) {
-      handler();
-      return () => {};
-    }
-    this.stopHandlers.add(handler);
-    return () => {
-      this.stopHandlers.delete(handler);
-    };
-  }
-
-  requirePgPool() {
-    if (this.pgwire_pool == null) {
-      throw new Error('No source connection configured');
-    } else {
-      return this.pgwire_pool!;
-    }
-  }
-}

package/src/util/PgManager.ts

@@ -1,64 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { NormalizedPostgresConnection } from '@powersync/service-types';
-
-export class PgManager {
-  /**
-   * Do not use this for any transactions.
-   */
-  public readonly pool: pgwire.PgClient;
-
-  private connectionPromises: Promise<pgwire.PgConnection>[] = [];
-
-  constructor(public options: NormalizedPostgresConnection, public poolOptions: pgwire.PgPoolOptions) {
-    // The pool is lazy - no connections are opened until a query is performed.
-    this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
-  }
-
-  /**
-   * Create a new replication connection.
-   */
-  async replicationConnection(): Promise<pgwire.PgConnection> {
-    const p = pgwire.connectPgWire(this.options, { type: 'replication' });
-    this.connectionPromises.push(p);
-    return await p;
-  }
-
-  /**
-   * Create a new standard connection, used for initial snapshot.
-   *
-   * This connection must not be shared between multiple async contexts.
-   */
-  async snapshotConnection(): Promise<pgwire.PgConnection> {
-    const p = pgwire.connectPgWire(this.options, { type: 'standard' });
-    this.connectionPromises.push(p);
-    return await p;
-  }
-
-  async end() {
-    for (let result of await Promise.allSettled([
-      this.pool.end(),
-      ...this.connectionPromises.map((promise) => {
-        return promise.then((connection) => connection.end());
-      })
-    ])) {
-      // Throw the first error, if any
-      if (result.status == 'rejected') {
-        throw result.reason;
-      }
-    }
-  }
-
-  async destroy() {
-    this.pool.destroy();
-    for (let result of await Promise.allSettled([
-      ...this.connectionPromises.map((promise) => {
-        return promise.then((connection) => connection.destroy());
-      })
-    ])) {
-      // Throw the first error, if any
-      if (result.status == 'rejected') {
-        throw result.reason;
-      }
-    }
-  }
-}

package/src/util/migration_lib.ts

@@ -1,79 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-
-export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
-
-interface Migration {
-  id: number;
-  name: string;
-  up: MigrationFunction;
-}
-
-// Very loosely based on https://github.com/porsager/postgres-shift/
-export class Migrations {
-  private migrations: Migration[] = [];
-
-  add(id: number, name: string, up: MigrationFunction) {
-    if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
-      throw new Error('Migration ids must be strictly incrementing');
-    }
-    this.migrations.push({ id, up, name });
-  }
-
-  async up(db: pgwire.PgConnection) {
-    await db.query('BEGIN');
-    try {
-      await this.ensureMigrationsTable(db);
-      const current = await this.getCurrentMigration(db);
-      let currentId = current ? current.id : 0;
-
-      for (let migration of this.migrations) {
-        if (migration.id <= currentId) {
-          continue;
-        }
-        await migration.up(db);
-
-        await db.query({
-          statement: `
-      insert into migrations (
-        migration_id,
-        name
-      ) values (
-        $1,
-        $2
-      )
-    `,
-          params: [
-            { type: 'int4', value: migration.id },
-            { type: 'varchar', value: migration.name }
-          ]
-        });
-      }
-
-      await db.query('COMMIT');
-    } catch (e) {
-      await db.query('ROLLBACK');
-      throw e;
-    }
-  }
-
-  getCurrentMigration(db: pgwire.PgConnection) {
-    return db
-      .query(
-        `
-      select migration_id as id from migrations
-      order by migration_id desc
-      limit 1
-    `
-      )
-      .then((results) => ({ id: results.rows[0][0] as number }));
-  }
-
-  async ensureMigrationsTable(db: pgwire.PgConnection) {
-    await db.query(`create table if not exists migrations (
-        migration_id serial primary key,
-        created_at timestamp with time zone not null default now(),
-        name text
-      )
-    `);
-  }
-}

package/src/util/pgwire_utils.ts

@@ -1,139 +0,0 @@
-// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
-
-import * as bson from 'bson';
-import * as uuid from 'uuid';
-import * as pgwire from '@powersync/service-jpgwire';
-import { SqliteJsonValue, SqliteRow, ToastableSqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
-
-import * as replication from '../replication/replication-index.js';
-import { logger } from '@powersync/lib-services-framework';
-
-/**
- * pgwire message -> SQLite row.
- * @param message
- */
-export function constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteRow {
-  const rawData = (message as any).afterRaw;
-
-  const record = pgwire.decodeTuple(message.relation, rawData);
-  return toSyncRulesRow(record);
-}
-
-export function hasToastedValues(row: ToastableSqliteRow) {
-  for (let key in row) {
-    if (typeof row[key] == 'undefined') {
-      return true;
-    }
-  }
-  return false;
-}
-
-export function isCompleteRow(row: ToastableSqliteRow): row is SqliteRow {
-  return !hasToastedValues(row);
-}
-
-/**
- * pgwire message -> SQLite row.
- * @param message
- */
-export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteRow | undefined {
-  const rawData = (message as any).beforeRaw;
-  if (rawData == null) {
-    return undefined;
-  }
-  const record = pgwire.decodeTuple(message.relation, rawData);
-  return toSyncRulesRow(record);
-}
-
-function getRawReplicaIdentity(
-  tuple: ToastableSqliteRow,
-  columns: replication.ReplicationColumn[]
-): Record<string, any> {
-  let result: Record<string, any> = {};
-  for (let column of columns) {
-    const name = column.name;
-    result[name] = tuple[name];
-  }
-  return result;
-}
-const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
-
-export function getUuidReplicaIdentityString(
-  tuple: ToastableSqliteRow,
-  columns: replication.ReplicationColumn[]
-): string {
-  const rawIdentity = getRawReplicaIdentity(tuple, columns);
-
-  return uuidForRow(rawIdentity);
-}
-
-export function uuidForRow(row: SqliteRow): string {
-  // Important: This must not change, since it will affect how ids are generated.
-  // Use BSON so that it's a well-defined format without encoding ambiguities.
-  const repr = bson.serialize(row);
-  return uuid.v5(repr, ID_NAMESPACE);
-}
-
-export function getUuidReplicaIdentityBson(
-  tuple: ToastableSqliteRow,
-  columns: replication.ReplicationColumn[]
-): bson.UUID {
-  if (columns.length == 0) {
-    // REPLICA IDENTITY NOTHING - generate random id
-    return new bson.UUID(uuid.v4());
-  }
-  const rawIdentity = getRawReplicaIdentity(tuple, columns);
-
-  return uuidForRowBson(rawIdentity);
-}
-
-export function uuidForRowBson(row: SqliteRow): bson.UUID {
-  // Important: This must not change, since it will affect how ids are generated.
-  // Use BSON so that it's a well-defined format without encoding ambiguities.
-  const repr = bson.serialize(row);
-  const buffer = Buffer.alloc(16);
-  return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
-}
-
-export function escapeIdentifier(identifier: string) {
-  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
-}
-
-export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam {
-  if (arg == null) {
-    return { type: 'varchar', value: null };
-  } else if (typeof arg == 'string') {
-    return { type: 'varchar', value: arg };
-  } else if (typeof arg == 'number') {
-    if (Number.isInteger(arg)) {
-      return { type: 'int8', value: arg };
-    } else {
-      return { type: 'float8', value: arg };
-    }
-  } else if (typeof arg == 'boolean') {
-    return { type: 'bool', value: arg };
-  } else if (typeof arg == 'bigint') {
-    return { type: 'int8', value: arg };
-  } else {
-    throw new Error(`Unsupported query parameter: ${typeof arg}`);
-  }
-}
-
-export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise<pgwire.PgResult>;
-export async function retriedQuery(db: pgwire.PgClient, query: string): Promise<pgwire.PgResult>;
-
-/**
- * Retry a simple query - up to 2 attempts total.
- */
-export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) {
-  for (let tries = 2; ; tries--) {
-    try {
-      return await db.query(...args);
-    } catch (e) {
-      if (tries == 1) {
-        throw e;
-      }
-      logger.warn('Query error, retrying', e);
-    }
-  }
-}

package/src/util/populate_test_data.ts

@@ -1,78 +0,0 @@
-import * as crypto from 'crypto';
-import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
-
-import { connectPgWire } from '@powersync/service-jpgwire';
-import { NormalizedPostgresConnection } from '@powersync/service-types';
-
-// This util is actually for tests only, but we need it compiled to JS for the service to work, so it's placed in the service.
-
-export interface PopulateDataOptions {
-  connection: NormalizedPostgresConnection;
-  num_transactions: number;
-  per_transaction: number;
-  size: number;
-}
-
-if (isMainThread || parentPort == null) {
-  // Not a worker - ignore
-} else {
-  try {
-    const options = workerData as PopulateDataOptions;
-
-    const result = await populateDataInner(options);
-    parentPort.postMessage(result);
-    process.exit(0);
-  } catch (e) {
-    // This is a bug, not a connection issue
-    console.error(e);
-    // Only closes the Worker thread
-    process.exit(2);
-  }
-}
-
-async function populateDataInner(options: PopulateDataOptions) {
-  // Dedicated connection so we can release the memory easily
-  const initialDb = await connectPgWire(options.connection, { type: 'standard' });
-  const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
-  let operation_count = 0;
-  for (let i = 0; i < options.num_transactions; i++) {
-    const prefix = `test${i}K`;
-
-    await initialDb.query({
-      statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
-      params: [
-        { type: 'varchar', value: prefix },
-        { type: 'varchar', value: largeDescription },
-        { type: 'int4', value: options.per_transaction }
-      ]
-    });
-    operation_count += options.per_transaction;
-  }
-  await initialDb.end();
-  return operation_count;
-}
-
-export async function populateData(options: PopulateDataOptions) {
-  const WORKER_TIMEOUT = 30_000;
-
-  const worker = new Worker(new URL('./populate_test_data.js', import.meta.url), {
-    workerData: options
-  });
-  const timeout = setTimeout(() => {
-    // Exits with code 1 below
-    worker.terminate();
-  }, WORKER_TIMEOUT);
-  try {
-    return await new Promise<number>((resolve, reject) => {
-      worker.on('message', resolve);
-      worker.on('error', reject);
-      worker.on('exit', (code) => {
-        if (code !== 0) {
-          reject(new Error(`Populating data failed with exit code ${code}`));
-        }
-      });
-    });
-  } finally {
-    clearTimeout(timeout);
-  }
-}

package/test/src/__snapshots__/pg_test.test.ts.snap

@@ -1,256 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`pg data types > schema 1`] = `
-[
-  {
-    "name": "public",
-    "tables": [
-      {
-        "columns": [
-          {
-            "name": "id",
-            "pg_type": "int4",
-            "type": "integer",
-          },
-          {
-            "name": "text",
-            "pg_type": "text",
-            "type": "text",
-          },
-          {
-            "name": "uuid",
-            "pg_type": "uuid",
-            "type": "uuid",
-          },
-          {
-            "name": "varchar",
-            "pg_type": "varchar",
-            "type": "character varying(255)",
-          },
-          {
-            "name": "bool",
-            "pg_type": "bool",
-            "type": "boolean",
-          },
-          {
-            "name": "bytea",
-            "pg_type": "bytea",
-            "type": "bytea",
-          },
-          {
-            "name": "int2",
-            "pg_type": "int2",
-            "type": "smallint",
-          },
-          {
-            "name": "int4",
-            "pg_type": "int4",
-            "type": "integer",
-          },
-          {
-            "name": "int8",
-            "pg_type": "int8",
-            "type": "bigint",
-          },
-          {
-            "name": "float4",
-            "pg_type": "float4",
-            "type": "real",
-          },
-          {
-            "name": "float8",
-            "pg_type": "float8",
-            "type": "double precision",
-          },
-          {
-            "name": "numeric",
-            "pg_type": "numeric",
-            "type": "numeric",
-          },
-          {
-            "name": "json",
-            "pg_type": "json",
-            "type": "json",
-          },
-          {
-            "name": "jsonb",
-            "pg_type": "jsonb",
-            "type": "jsonb",
-          },
-          {
-            "name": "pg_lsn",
-            "pg_type": "pg_lsn",
-            "type": "pg_lsn",
-          },
-          {
-            "name": "date",
-            "pg_type": "date",
-            "type": "date",
-          },
-          {
-            "name": "time",
-            "pg_type": "time",
-            "type": "time without time zone",
-          },
-          {
-            "name": "timestamp",
-            "pg_type": "timestamp",
-            "type": "timestamp without time zone",
-          },
-          {
-            "name": "timestamptz",
-            "pg_type": "timestamptz",
-            "type": "timestamp with time zone",
-          },
-          {
-            "name": "interval",
-            "pg_type": "interval",
-            "type": "interval",
-          },
-          {
-            "name": "macaddr",
-            "pg_type": "macaddr",
-            "type": "macaddr",
-          },
-          {
-            "name": "inet",
-            "pg_type": "inet",
-            "type": "inet",
-          },
-          {
-            "name": "oid",
-            "pg_type": "oid",
-            "type": "oid",
-          },
-        ],
-        "name": "test_data",
-      },
-      {
-        "columns": [
-          {
-            "name": "id",
-            "pg_type": "int4",
-            "type": "integer",
-          },
-          {
-            "name": "text",
-            "pg_type": "text[]",
-            "type": "text[]",
-          },
-          {
-            "name": "uuid",
-            "pg_type": "uuid[]",
-            "type": "uuid[]",
-          },
-          {
-            "name": "varchar",
-            "pg_type": "varchar[]",
-            "type": "character varying(255)[]",
-          },
-          {
-            "name": "bool",
-            "pg_type": "bool[]",
-            "type": "boolean[]",
-          },
-          {
-            "name": "bytea",
-            "pg_type": "bytea[]",
-            "type": "bytea[]",
-          },
-          {
-            "name": "int2",
-            "pg_type": "int2[]",
-            "type": "smallint[]",
-          },
-          {
-            "name": "int4",
-            "pg_type": "int4[]",
-            "type": "integer[]",
-          },
-          {
-            "name": "int8",
-            "pg_type": "int8[]",
-            "type": "bigint[]",
-          },
-          {
-            "name": "float4",
-            "pg_type": "float4[]",
-            "type": "real[]",
-          },
-          {
-            "name": "float8",
-            "pg_type": "float8[]",
-            "type": "double precision[]",
-          },
-          {
-            "name": "numeric",
-            "pg_type": "numeric[]",
-            "type": "numeric[]",
-          },
-          {
-            "name": "json",
-            "pg_type": "json[]",
-            "type": "json[]",
-          },
-          {
-            "name": "jsonb",
-            "pg_type": "jsonb[]",
-            "type": "jsonb[]",
-          },
-          {
-            "name": "pg_lsn",
-            "pg_type": "pg_lsn[]",
-            "type": "pg_lsn[]",
-          },
-          {
-            "name": "date",
-            "pg_type": "date[]",
-            "type": "date[]",
-          },
-          {
-            "name": "time",
-            "pg_type": "time[]",
-            "type": "time without time zone[]",
-          },
-          {
-            "name": "timestamp",
-            "pg_type": "timestamp[]",
-            "type": "timestamp without time zone[]",
-          },
-          {
-            "name": "timestamptz",
-            "pg_type": "timestamptz[]",
-            "type": "timestamp with time zone[]",
-          },
-          {
-            "name": "interval",
-            "pg_type": "interval[]",
-            "type": "interval[]",
-          },
-          {
-            "name": "macaddr",
-            "pg_type": "macaddr[]",
-            "type": "macaddr[]",
-          },
-          {
-            "name": "inet",
-            "pg_type": "inet[]",
-            "type": "inet[]",
-          },
-          {
-            "name": "oid",
-            "pg_type": "oid[]",
-            "type": "oid[]",
-          },
-          {
-            "name": "multidimensional",
-            "pg_type": "text[]",
-            "type": "text[]",
-          },
-        ],
-        "name": "test_data_arrays",
-      },
-    ],
-  },
-]
-`;