@powersync/service-core 0.0.2

package/src/util/migration_lib.ts

@@ -0,0 +1,79 @@
+import * as pgwire from '@powersync/service-jpgwire';
+
+export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
+
+interface Migration {
+  id: number;
+  name: string;
+  up: MigrationFunction;
+}
+
+// Very loosely based on https://github.com/porsager/postgres-shift/
+export class Migrations {
+  private migrations: Migration[] = [];
+
+  add(id: number, name: string, up: MigrationFunction) {
+    if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
+      throw new Error('Migration ids must be strictly incrementing');
+    }
+    this.migrations.push({ id, up, name });
+  }
+
+  async up(db: pgwire.PgConnection) {
+    await db.query('BEGIN');
+    try {
+      await this.ensureMigrationsTable(db);
+      const current = await this.getCurrentMigration(db);
+      let currentId = current ? current.id : 0;
+
+      for (let migration of this.migrations) {
+        if (migration.id <= currentId) {
+          continue;
+        }
+        await migration.up(db);
+
+        await db.query({
+          statement: `
+      insert into migrations (
+        migration_id,
+        name
+      ) values (
+        $1,
+        $2
+      )
+    `,
+          params: [
+            { type: 'int4', value: migration.id },
+            { type: 'varchar', value: migration.name }
+          ]
+        });
+      }
+
+      await db.query('COMMIT');
+    } catch (e) {
+      await db.query('ROLLBACK');
+      throw e;
+    }
+  }
+
+  getCurrentMigration(db: pgwire.PgConnection) {
+    return db
+      .query(
+        `
+      select migration_id as id from migrations
+      order by migration_id desc
+      limit 1
+    `
+      )
+      .then((results) => ({ id: results.rows[0][0] as number }));
+  }
+
+  async ensureMigrationsTable(db: pgwire.PgConnection) {
+    await db.query(`create table if not exists migrations (
+        migration_id serial primary key,
+        created_at timestamp with time zone not null default now(),
+        name text
+      )
+    `);
+  }
+}

package/src/util/pgwire_utils.ts

@@ -0,0 +1,139 @@
+// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
+
+import * as bson from 'bson';
+import * as uuid from 'uuid';
+import * as pgwire from '@powersync/service-jpgwire';
+import { SqliteJsonValue, SqliteRow, ToastableSqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
+import * as micro from '@journeyapps-platform/micro';
+
+import * as replication from '@/replication/replication-index.js';
+
+/**
+ * pgwire message -> SQLite row.
+ * @param message
+ */
+export function constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteRow {
+  const rawData = (message as any).afterRaw;
+
+  const record = pgwire.decodeTuple(message.relation, rawData);
+  return toSyncRulesRow(record);
+}
+
+export function hasToastedValues(row: ToastableSqliteRow) {
+  for (let key in row) {
+    if (typeof row[key] == 'undefined') {
+      return true;
+    }
+  }
+  return false;
+}
+
+export function isCompleteRow(row: ToastableSqliteRow): row is SqliteRow {
+  return !hasToastedValues(row);
+}
+
+/**
+ * pgwire message -> SQLite row.
+ * @param message
+ */
+export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteRow | undefined {
+  const rawData = (message as any).beforeRaw;
+  if (rawData == null) {
+    return undefined;
+  }
+  const record = pgwire.decodeTuple(message.relation, rawData);
+  return toSyncRulesRow(record);
+}
+
+function getRawReplicaIdentity(
+  tuple: ToastableSqliteRow,
+  columns: replication.ReplicationColumn[]
+): Record<string, any> {
+  let result: Record<string, any> = {};
+  for (let column of columns) {
+    const name = column.name;
+    result[name] = tuple[name];
+  }
+  return result;
+}
+const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
+
+export function getUuidReplicaIdentityString(
+  tuple: ToastableSqliteRow,
+  columns: replication.ReplicationColumn[]
+): string {
+  const rawIdentity = getRawReplicaIdentity(tuple, columns);
+
+  return uuidForRow(rawIdentity);
+}
+
+export function uuidForRow(row: SqliteRow): string {
+  // Important: This must not change, since it will affect how ids are generated.
+  // Use BSON so that it's a well-defined format without encoding ambiguities.
+  const repr = bson.serialize(row);
+  return uuid.v5(repr, ID_NAMESPACE);
+}
+
+export function getUuidReplicaIdentityBson(
+  tuple: ToastableSqliteRow,
+  columns: replication.ReplicationColumn[]
+): bson.UUID {
+  if (columns.length == 0) {
+    // REPLICA IDENTITY NOTHING - generate random id
+    return new bson.UUID(uuid.v4());
+  }
+  const rawIdentity = getRawReplicaIdentity(tuple, columns);
+
+  return uuidForRowBson(rawIdentity);
+}
+
+export function uuidForRowBson(row: SqliteRow): bson.UUID {
+  // Important: This must not change, since it will affect how ids are generated.
+  // Use BSON so that it's a well-defined format without encoding ambiguities.
+  const repr = bson.serialize(row);
+  const buffer = Buffer.alloc(16);
+  return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
+}
+
+export function escapeIdentifier(identifier: string) {
+  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
+}
+
+export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam {
+  if (arg == null) {
+    return { type: 'varchar', value: null };
+  } else if (typeof arg == 'string') {
+    return { type: 'varchar', value: arg };
+  } else if (typeof arg == 'number') {
+    if (Number.isInteger(arg)) {
+      return { type: 'int8', value: arg };
+    } else {
+      return { type: 'float8', value: arg };
+    }
+  } else if (typeof arg == 'boolean') {
+    return { type: 'bool', value: arg };
+  } else if (typeof arg == 'bigint') {
+    return { type: 'int8', value: arg };
+  } else {
+    throw new Error(`Unsupported query parameter: ${typeof arg}`);
+  }
+}
+
+export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise<pgwire.PgResult>;
+export async function retriedQuery(db: pgwire.PgClient, query: string): Promise<pgwire.PgResult>;
+
+/**
+ * Retry a simple query - up to 2 attempts total.
+ */
+export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) {
+  for (let tries = 2; ; tries--) {
+    try {
+      return await db.query(...args);
+    } catch (e) {
+      if (tries == 1) {
+        throw e;
+      }
+      micro.logger.warn('Query error, retrying', e);
+    }
+  }
+}

package/src/util/populate_test_data.ts

@@ -0,0 +1,78 @@
+import * as crypto from 'crypto';
+import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
+
+import { connectPgWire } from '@powersync/service-jpgwire';
+import { NormalizedPostgresConnection } from '@powersync/service-types';
+
+// This util is actually for tests only, but we need it compiled to JS for the service to work, so it's placed in the service.
+
+export interface PopulateDataOptions {
+  connection: NormalizedPostgresConnection;
+  num_transactions: number;
+  per_transaction: number;
+  size: number;
+}
+
+if (isMainThread || parentPort == null) {
+  // Not a worker - ignore
+} else {
+  try {
+    const options = workerData as PopulateDataOptions;
+
+    const result = await populateDataInner(options);
+    parentPort.postMessage(result);
+    process.exit(0);
+  } catch (e) {
+    // This is a bug, not a connection issue
+    console.error(e);
+    // Only closes the Worker thread
+    process.exit(2);
+  }
+}
+
+async function populateDataInner(options: PopulateDataOptions) {
+  // Dedicated connection so we can release the memory easily
+  const initialDb = await connectPgWire(options.connection, { type: 'standard' });
+  const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
+  let operation_count = 0;
+  for (let i = 0; i < options.num_transactions; i++) {
+    const prefix = `test${i}K`;
+
+    await initialDb.query({
+      statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
+      params: [
+        { type: 'varchar', value: prefix },
+        { type: 'varchar', value: largeDescription },
+        { type: 'int4', value: options.per_transaction }
+      ]
+    });
+    operation_count += options.per_transaction;
+  }
+  await initialDb.end();
+  return operation_count;
+}
+
+export async function populateData(options: PopulateDataOptions) {
+  const WORKER_TIMEOUT = 30_000;
+
+  const worker = new Worker(new URL('./populate_test_data.js', import.meta.url), {
+    workerData: options
+  });
+  const timeout = setTimeout(() => {
+    // Exits with code 1 below
+    worker.terminate();
+  }, WORKER_TIMEOUT);
+  try {
+    return await new Promise<number>((resolve, reject) => {
+      worker.on('message', resolve);
+      worker.on('error', reject);
+      worker.on('exit', (code) => {
+        if (code !== 0) {
+          reject(new Error(`Populating data failed with exit code ${code}`));
+        }
+      });
+    });
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/src/util/protocol-types.ts

@@ -0,0 +1,223 @@
+import * as t from 'ts-codec';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+
+/**
+ * For sync2.json
+ */
+export interface ContinueCheckpointRequest {
+  /**
+   * Existing bucket states. Only these buckets are synchronized.
+   */
+  buckets: BucketRequest[];
+
+  checkpoint_token: string;
+
+  limit?: number;
+}
+
+export interface SyncNewCheckpointRequest {
+  /**
+   * Existing bucket states. Used if include_data is specified.
+   */
+  buckets?: BucketRequest[];
+
+  request_checkpoint: {
+    /**
+     * Whether or not to include an initial data request.
+     */
+    include_data: boolean;
+
+    /**
+     * Whether or not to compute a checksum.
+     */
+    include_checksum: boolean;
+  };
+
+  limit?: number;
+}
+
+export type SyncRequest = ContinueCheckpointRequest | SyncNewCheckpointRequest;
+
+export interface SyncResponse {
+  /**
+   * Data for the buckets returned. May not have an an entry for each bucket in the request.
+   */
+  data?: SyncBucketData[];
+
+  /**
+   * True if the response limit has been reached, and another request must be made.
+   */
+  has_more: boolean;
+
+  checkpoint_token?: string;
+
+  checkpoint?: Checkpoint;
+}
+
+export const BucketRequest = t.object({
+  name: t.string,
+
+  /**
+   * Base-10 number. Sync all data from this bucket with op_id > after.
+   */
+  after: t.string
+});
+
+export type BucketRequest = t.Decoded<typeof BucketRequest>;
+
+export const StreamingSyncRequest = t.object({
+  /**
+   * Existing bucket states.
+   */
+  buckets: t.array(BucketRequest).optional(),
+
+  /**
+   * If specified, limit the response to only include these buckets.
+   */
+  only: t.array(t.string).optional(),
+
+  /**
+   * Whether or not to compute a checksum for each checkpoint
+   */
+  include_checksum: t.boolean.optional(),
+
+  /**
+   * True to keep `data` as a string, instead of nested JSON.
+   */
+  raw_data: t.boolean.optional(),
+
+  /**
+   * Data is received in a serialized BSON Buffer
+   */
+  binary_data: t.boolean.optional()
+});
+
+export type StreamingSyncRequest = t.Decoded<typeof StreamingSyncRequest>;
+
+export interface StreamingSyncCheckpoint {
+  checkpoint: Checkpoint;
+}
+
+export interface StreamingSyncCheckpointDiff {
+  checkpoint_diff: {
+    last_op_id: OpId;
+    write_checkpoint?: OpId;
+    updated_buckets: BucketChecksum[];
+    removed_buckets: string[];
+  };
+}
+
+export interface StreamingSyncData {
+  data: SyncBucketData;
+}
+
+export interface StreamingSyncCheckpointComplete {
+  checkpoint_complete: {
+    last_op_id: OpId;
+  };
+}
+
+export interface StreamingSyncKeepalive {}
+
+export type StreamingSyncLine =
+  | StreamingSyncData
+  | StreamingSyncCheckpoint
+  | StreamingSyncCheckpointDiff
+  | StreamingSyncCheckpointComplete
+  | StreamingSyncKeepalive;
+
+/**
+ * 64-bit unsigned number, as a base-10 string.
+ */
+export type OpId = string;
+
+export interface Checkpoint {
+  last_op_id: OpId;
+  write_checkpoint?: OpId;
+  buckets: BucketChecksum[];
+}
+
+export interface BucketState {
+  bucket: string;
+  op_id: string;
+}
+
+export interface SyncDataBatch {
+  buckets: SyncBucketData[];
+}
+
+export interface SyncBucketData {
+  bucket: string;
+  data: OplogEntry[];
+  /**
+   * True if the response does not contain all the data for this bucket, and another request must be made.
+   */
+  has_more: boolean;
+  /**
+   * The `after` specified in the request.
+   */
+  after: OpId;
+  /**
+   * Use this for the next request.
+   */
+  next_after: OpId;
+}
+
+export interface OplogEntry {
+  op_id: OpId;
+  op: 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+  object_type?: string;
+  object_id?: string;
+  data?: Record<string, SqliteJsonValue> | string | null;
+  checksum: number | bigint;
+  subkey?: string;
+}
+
+export interface BucketChecksum {
+  bucket: string;
+  /**
+   * 32-bit unsigned hash.
+   */
+  checksum: number;
+
+  /**
+   * Count of operations - informational only.
+   */
+  count: number;
+}
+
+export function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest {
+  return (
+    Array.isArray((request as ContinueCheckpointRequest).buckets) &&
+    typeof (request as ContinueCheckpointRequest).checkpoint_token == 'string'
+  );
+}
+
+export function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest {
+  return typeof (request as SyncNewCheckpointRequest).request_checkpoint == 'object';
+}
+
+/**
+ * For crud.json
+ */
+export interface CrudRequest {
+  data: CrudEntry[];
+}
+
+export interface CrudEntry {
+  op: 'PUT' | 'PATCH' | 'DELETE';
+  type: string;
+  id: string;
+  data: string;
+}
+
+export interface CrudResponse {
+  /**
+   * A sync response with a checkpoint >= this checkpoint would contain all the changes in this request.
+   *
+   * Any earlier checkpoint may or may not contain these changes.
+   *
+   * May be empty when the request contains no ops.
+   */
+  checkpoint?: OpId;
+}

package/src/util/secs.ts

@@ -0,0 +1,54 @@
+// From https://github.com/panva/jose/blob/1c2ae17a15838757ae2bfda9d1d8389d435f74b5/src/lib/secs.ts#L8
+// MIT license.
+// Copied here since it's not exported by jose.
+
+const minute = 60;
+const hour = minute * 60;
+const day = hour * 24;
+const week = day * 7;
+const year = day * 365.25;
+
+const REGEX = /^(\d+|\d+\.\d+) ?(seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)$/i;
+
+export default (str: string): number => {
+  const matched = REGEX.exec(str);
+
+  if (!matched) {
+    throw new TypeError('Invalid time period format');
+  }
+
+  const value = parseFloat(matched[1]);
+  const unit = matched[2].toLowerCase();
+
+  switch (unit) {
+    case 'sec':
+    case 'secs':
+    case 'second':
+    case 'seconds':
+    case 's':
+      return Math.round(value);
+    case 'minute':
+    case 'minutes':
+    case 'min':
+    case 'mins':
+    case 'm':
+      return Math.round(value * minute);
+    case 'hour':
+    case 'hours':
+    case 'hr':
+    case 'hrs':
+    case 'h':
+      return Math.round(value * hour);
+    case 'day':
+    case 'days':
+    case 'd':
+      return Math.round(value * day);
+    case 'week':
+    case 'weeks':
+    case 'w':
+      return Math.round(value * week);
+    // years matched
+    default:
+      return Math.round(value * year);
+  }
+};

package/src/util/util-index.ts

@@ -0,0 +1,25 @@
+export * from './alerting.js';
+export * from './env.js';
+export * from './memory-tracking.js';
+export * from './migration_lib.js';
+export * from './Mutex.js';
+export * from './PgManager.js';
+export * from './pgwire_utils.js';
+export * from './populate_test_data.js';
+export * from './protocol-types.js';
+export * from './secs.js';
+export * from './utils.js';
+
+export * from './config.js';
+export * from './config/types.js';
+export * from './config/compound-config-collector.js';
+
+export * from './config/collectors/config-collector.js';
+export * from './config/collectors/impl/base64-config-collector.js';
+export * from './config/collectors/impl/fallback-config-collector.js';
+export * from './config/collectors/impl/filesystem-config-collector.js';
+
+export * from './config/sync-rules/sync-collector.js';
+export * from './config/sync-rules/impl/base64-sync-rules-collector.js';
+export * from './config/sync-rules/impl/filesystem-sync-rules-collector.js';
+export * from './config/sync-rules/impl/inline-sync-rules-collector.js';

package/src/util/utils.ts

@@ -0,0 +1,102 @@
+import crypto from 'crypto';
+import * as pgwire from '@powersync/service-jpgwire';
+import { pgwireRows } from '@powersync/service-jpgwire';
+import * as micro from '@journeyapps-platform/micro';
+
+import * as storage from '@/storage/storage-index.js';
+import { BucketChecksum, OpId } from './protocol-types.js';
+import { retriedQuery } from './pgwire_utils.js';
+
+export function hashData(type: string, id: string, data: string): number {
+  const hash = crypto.createHash('sha256');
+  hash.update(`put.${type}.${id}.${data}`);
+  const buffer = hash.digest();
+  return buffer.readUInt32LE(0);
+}
+
+export function hashDelete(sourceKey: string) {
+  const hash = crypto.createHash('sha256');
+  hash.update(`delete.${sourceKey}`);
+  const buffer = hash.digest();
+  return buffer.readUInt32LE(0);
+}
+
+export function timestampToOpId(ts: bigint): OpId {
+  // Dynamic values are passed in in some cases, so we make extra sure that the
+  // number is a bigint and not number or Long.
+  if (typeof ts != 'bigint') {
+    throw new Error(`bigint expected, got: ${ts} (${typeof ts})`);
+  }
+  return ts.toString(10);
+}
+
+export function checksumsDiff(previous: BucketChecksum[], current: BucketChecksum[]) {
+  const updated_buckets: BucketChecksum[] = [];
+
+  const previousBuckets = new Map<string, BucketChecksum>();
+  for (let checksum of previous) {
+    previousBuckets.set(checksum.bucket, checksum);
+  }
+  for (let checksum of current) {
+    if (!previousBuckets.has(checksum.bucket)) {
+      updated_buckets.push(checksum);
+    } else {
+      const p = previousBuckets.get(checksum.bucket);
+      if (p?.checksum != checksum.checksum || p?.count != checksum.count) {
+        updated_buckets.push(checksum);
+      }
+      previousBuckets.delete(checksum.bucket);
+    }
+  }
+
+  const removed_buckets: string[] = [...previousBuckets.keys()];
+  return {
+    updated_buckets,
+    removed_buckets
+  };
+}
+
+export async function getClientCheckpoint(
+  db: pgwire.PgClient,
+  bucketStorage: storage.BucketStorageFactory,
+  options?: { timeout?: number }
+): Promise<OpId> {
+  const start = Date.now();
+
+  const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
+
+  // This old API needs a persisted checkpoint id.
+  // Since we don't use LSNs anymore, the only way to get that is to wait.
+
+  const timeout = options?.timeout ?? 50_000;
+
+  micro.logger.info(`Waiting for LSN checkpoint: ${lsn}`);
+  while (Date.now() - start < timeout) {
+    const cp = await bucketStorage.getActiveCheckpoint();
+    if (!cp.hasSyncRules()) {
+      throw new Error('No sync rules available');
+    }
+    if (cp.lsn >= lsn) {
+      micro.logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
+      return cp.checkpoint;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 30));
+  }
+
+  throw new Error('Timeout while waiting for checkpoint');
+}
+
+export async function createWriteCheckpoint(
+  db: pgwire.PgClient,
+  bucketStorage: storage.BucketStorageFactory,
+  user_id: string
+): Promise<bigint> {
+  const [{ lsn }] = pgwireRows(
+    await retriedQuery(db, `SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`)
+  );
+
+  const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
+  micro.logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
+  return id;
+}