@powersync/service-module-postgres-storage 0.0.0-dev-20250116115804

package/src/types/codecs.ts

@@ -0,0 +1,136 @@
+import * as t from 'ts-codec';
+
+export const BIGINT_MAX = BigInt('9223372036854775807');
+
+/**
+ * The use of ts-codec:
+ * We currently use pgwire for Postgres queries. This library provides fine-grained control
+ * over parameter typings and efficient streaming of query responses. Additionally, configuring
+ * pgwire with default certificates allows us to use the same connection configuration process
+ * for both replication and storage libraries.
+ *
+ * Unfortunately, ORM driver support for pgwire is limited, so we rely on pure SQL queries in the
+ * absence of writing an ORM driver from scratch.
+ *
+ * [Opinion]: Writing pure SQL queries throughout a codebase can be daunting from a maintenance
+ * and debugging perspective. For example, row response types are often declared when performing a query:
+ *
+ * ```typescript
+ * const rows = await db.queryRows<MyRowType>(`SELECT one, two FROM my_table`);
+ * ```
+ * This type declaration suggests `rows` is an array of `MyRowType` objects, even though no validation
+ * is enforced. Adding a field to the `MyRowType` interface without updating the query could easily
+ * introduce subtle bugs. Similarly, type mismatches between SQL results and TypeScript interfaces, such as
+ * a `Date` field returned as a `string`, require manual conversion.
+ *
+ * `ts-codec` is not an ORM, but it simplifies working with pure SQL query responses in several ways:
+ *
+ * - **Validations**: The `decode` operation ensures that the returned row matches the expected object
+ *   structure, throwing an error if it doesn't.
+ * - **Decoding Columns**: pgwire already decodes common SQLite types, but `ts-codec` adds an extra layer
+ *   for JS-native values. For instance, `jsonb` columns are returned as `JsonContainer`/`string` and can
+ *   be automatically parsed into objects. Similarly, fields like `group_id` are converted from `Bigint`
+ *   to `Number` for easier use.
+ * - **Encoded Forms**: A single `ts-codec` type definition can infer both encoded and decoded forms. This
+ *   is especially useful for persisted batch operations that rely on JSON query parameters for bulk inserts.
+ *   Collections like `bucket_data`, `current_data`, and `bucket_parameters` use encoded/decoded types, making
+ *   changes easier to manage and validate. While some manual encoding is done for intermediate values (e.g.,
+ *   size estimation), these types are validated with `ts-codec` to ensure consistency.
+ */
+
+/**
+ * Wraps a codec which is encoded to a JSON string
+ */
+export const jsonb = <Decoded>(subCodec: t.Codec<Decoded, any>) =>
+  t.codec<Decoded, string>(
+    'jsonb',
+    (decoded: Decoded) => {
+      return JSON.stringify(subCodec.encode(decoded) as any);
+    },
+    (encoded: string | { data: string }) => {
+      const s = typeof encoded == 'object' ? encoded.data : encoded;
+      return subCodec.decode(JSON.parse(s));
+    }
+  );
+
+/**
+ * Just performs a pure JSON.parse for the decoding step
+ */
+export const jsonb_raw = <Decoded>() =>
+  t.codec<Decoded, string>(
+    'jsonb_raw',
+    (decoded: Decoded) => {
+      return JSON.stringify(decoded);
+    },
+    (encoded: string | { data: string }) => {
+      const s = typeof encoded == 'object' ? encoded.data : encoded;
+      return JSON.parse(s);
+    }
+  );
+
+export const bigint = t.codec<bigint, string | number>(
+  'bigint',
+  (decoded: BigInt) => {
+    return decoded.toString();
+  },
+  (encoded: string | number) => {
+    return BigInt(encoded);
+  }
+);
+
+export const uint8array = t.codec<Uint8Array, Uint8Array>(
+  'uint8array',
+  (d) => d,
+  (e) => e
+);
+
+/**
+ * PGWire returns BYTEA values as Uint8Array instances.
+ * We also serialize to a hex string for bulk inserts.
+ */
+export const hexBuffer = t.codec(
+  'hexBuffer',
+  (decoded: Buffer) => {
+    return decoded.toString('hex');
+  },
+  (encoded: string | Uint8Array) => {
+    if (encoded instanceof Uint8Array) {
+      return Buffer.from(encoded);
+    }
+    if (typeof encoded !== 'string') {
+      throw new Error(`Expected either a Buffer instance or hex encoded buffer string`);
+    }
+    return Buffer.from(encoded, 'hex');
+  }
+);
+
+/**
+ * PGWire returns INTEGER columns as a `bigint`.
+ * This does a decode operation to `number`.
+ */
+export const pgwire_number = t.codec(
+  'pg_number',
+  (decoded: number) => decoded,
+  (encoded: bigint | number) => {
+    if (typeof encoded == 'number') {
+      return encoded;
+    }
+    if (typeof encoded !== 'bigint') {
+      throw new Error(`Expected either number or bigint for value`);
+    }
+    if (encoded > BigInt(Number.MAX_SAFE_INTEGER) || encoded < BigInt(Number.MIN_SAFE_INTEGER)) {
+      throw new RangeError('BigInt value is out of safe integer range for conversion to Number.');
+    }
+    return Number(encoded);
+  }
+);
+
+/**
+ * A codec which contains the same type on the input and output.
+ */
+export const IdentityCodec = <T>() =>
+  t.codec<T, T>(
+    'identity',
+    (encoded) => encoded,
+    (decoded) => decoded
+  );

package/src/types/models/ActiveCheckpoint.ts

@@ -0,0 +1,15 @@
+import * as t from 'ts-codec';
+import { bigint, pgwire_number } from '../codecs.js';
+
+/**
+ * Notification payload sent via Postgres' NOTIFY API.
+ *
+ */
+export const ActiveCheckpoint = t.object({
+  id: pgwire_number,
+  last_checkpoint: t.Null.or(bigint),
+  last_checkpoint_lsn: t.Null.or(t.string)
+});
+
+export type ActiveCheckpoint = t.Encoded<typeof ActiveCheckpoint>;
+export type ActiveCheckpointDecoded = t.Decoded<typeof ActiveCheckpoint>;

package/src/types/models/ActiveCheckpointNotification.ts

@@ -0,0 +1,14 @@
+import * as t from 'ts-codec';
+import { jsonb } from '../codecs.js';
+import { ActiveCheckpoint } from './ActiveCheckpoint.js';
+
+export const ActiveCheckpointPayload = t.object({
+  active_checkpoint: ActiveCheckpoint
+});
+
+export type ActiveCheckpointPayload = t.Encoded<typeof ActiveCheckpointPayload>;
+export type ActiveCheckpointPayloadDecoded = t.Decoded<typeof ActiveCheckpointPayload>;
+
+export const ActiveCheckpointNotification = jsonb(ActiveCheckpointPayload);
+export type ActiveCheckpointNotification = t.Encoded<typeof ActiveCheckpointNotification>;
+export type ActiveCheckpointNotificationDecoded = t.Decoded<typeof ActiveCheckpointNotification>;

package/src/types/models/BucketData.ts

@@ -0,0 +1,26 @@
+import * as t from 'ts-codec';
+import { bigint, hexBuffer, pgwire_number } from '../codecs.js';
+
+export enum OpType {
+  PUT = 'PUT',
+  REMOVE = 'REMOVE',
+  MOVE = 'MOVE',
+  CLEAR = 'CLEAR'
+}
+
+export const BucketData = t.object({
+  group_id: pgwire_number,
+  bucket_name: t.string,
+  op_id: bigint,
+  op: t.Enum(OpType),
+  source_table: t.Null.or(t.string),
+  source_key: t.Null.or(hexBuffer),
+  table_name: t.string.or(t.Null),
+  row_id: t.string.or(t.Null),
+  checksum: bigint,
+  data: t.Null.or(t.string),
+  target_op: t.Null.or(bigint)
+});
+
+export type BucketData = t.Encoded<typeof BucketData>;
+export type BucketDataDecoded = t.Decoded<typeof BucketData>;

package/src/types/models/BucketParameters.ts

@@ -0,0 +1,14 @@
+import * as t from 'ts-codec';
+import { bigint, hexBuffer, pgwire_number } from '../codecs.js';
+
+export const BucketParameters = t.object({
+  id: bigint,
+  group_id: pgwire_number,
+  source_table: t.string,
+  source_key: hexBuffer,
+  lookup: hexBuffer,
+  bucket_parameters: t.string
+});
+
+export type BucketParameters = t.Encoded<typeof BucketParameters>;
+export type BucketParametersDecoded = t.Decoded<typeof BucketParameters>;

package/src/types/models/CurrentData.ts

@@ -0,0 +1,23 @@
+import * as t from 'ts-codec';
+import { hexBuffer, jsonb, pgwire_number } from '../codecs.js';
+
+export const CurrentBucket = t.object({
+  bucket: t.string,
+  table: t.string,
+  id: t.string
+});
+
+export type CurrentBucket = t.Encoded<typeof CurrentBucket>;
+export type CurrentBucketDecoded = t.Decoded<typeof CurrentBucket>;
+
+export const CurrentData = t.object({
+  buckets: jsonb(t.array(CurrentBucket)),
+  data: hexBuffer,
+  group_id: pgwire_number,
+  lookups: t.array(hexBuffer),
+  source_key: hexBuffer,
+  source_table: t.string
+});
+
+export type CurrentData = t.Encoded<typeof CurrentData>;
+export type CurrentDataDecoded = t.Decoded<typeof CurrentData>;

package/src/types/models/Instance.ts

@@ -0,0 +1,8 @@
+import * as t from 'ts-codec';
+
+export const Instance = t.object({
+  id: t.string
+});
+
+export type Instance = t.Encoded<typeof Instance>;
+export type InstanceDecoded = t.Decoded<typeof Instance>;

package/src/types/models/Migration.ts

@@ -0,0 +1,19 @@
+import { framework } from '@powersync/service-core';
+import * as t from 'ts-codec';
+import { jsonb } from '../codecs.js';
+
+export const Migration = t.object({
+  last_run: t.string,
+  log: jsonb(
+    t.array(
+      t.object({
+        name: t.string,
+        direction: t.Enum(framework.migrations.Direction),
+        timestamp: framework.codecs.date
+      })
+    )
+  )
+});
+
+export type Migration = t.Encoded<typeof Migration>;
+export type MigrationDecoded = t.Decoded<typeof Migration>;

package/src/types/models/SourceTable.ts

@@ -0,0 +1,32 @@
+import * as t from 'ts-codec';
+import { bigint, jsonb, jsonb_raw, pgwire_number } from '../codecs.js';
+
+export type StoredRelationId = {
+  object_id: string | number;
+};
+
+export const ColumnDescriptor = t.object({
+  name: t.string,
+  /**
+   *  The type of the column ie VARCHAR, INT, etc
+   */
+  type: t.string.optional(),
+  /**
+   *  Some data sources have a type id that can be used to identify the type of the column
+   */
+  typeId: t.number.optional()
+});
+
+export const SourceTable = t.object({
+  id: t.string,
+  group_id: pgwire_number,
+  connection_id: bigint,
+  relation_id: t.Null.or(jsonb_raw<StoredRelationId>()),
+  schema_name: t.string,
+  table_name: t.string,
+  replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))),
+  snapshot_done: t.boolean
+});
+
+export type SourceTable = t.Encoded<typeof SourceTable>;
+export type SourceTableDecoded = t.Decoded<typeof SourceTable>;

package/src/types/models/SyncRules.ts

@@ -0,0 +1,50 @@
+import { framework, storage } from '@powersync/service-core';
+import * as t from 'ts-codec';
+import { bigint, pgwire_number } from '../codecs.js';
+
+export const SyncRules = t.object({
+  id: pgwire_number,
+  state: t.Enum(storage.SyncRuleState),
+  /**
+   * True if initial snapshot has been replicated.
+   *
+   * Can only be false if state == PROCESSING.
+   */
+  snapshot_done: t.boolean,
+  /**
+   * The last consistent checkpoint.
+   *
+   * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
+   */
+  last_checkpoint: t.Null.or(bigint),
+  /**
+   * The LSN associated with the last consistent checkpoint.
+   */
+  last_checkpoint_lsn: t.Null.or(t.string),
+  /**
+   * If set, no new checkpoints may be created < this value.
+   */
+  no_checkpoint_before: t.Null.or(t.string),
+  slot_name: t.string,
+  /**
+   * Last time we persisted a checkpoint.
+   *
+   * This may be old if no data is incoming.
+   */
+  last_checkpoint_ts: t.Null.or(framework.codecs.date),
+  /**
+   * Last time we persisted a checkpoint or keepalive.
+   *
+   * This should stay fairly current while replicating.
+   */
+  last_keepalive_ts: t.Null.or(framework.codecs.date),
+  /**
+   * If an error is stopping replication, it will be stored here.
+   */
+  last_fatal_error: t.Null.or(t.string),
+  keepalive_op: t.Null.or(bigint),
+  content: t.string
+});
+
+export type SyncRules = t.Encoded<typeof SyncRules>;
+export type SyncRulesDecoded = t.Decoded<typeof SyncRules>;

package/src/types/models/WriteCheckpoint.ts

@@ -0,0 +1,20 @@
+import * as t from 'ts-codec';
+import { bigint, jsonb } from '../codecs.js';
+
+export const WriteCheckpoint = t.object({
+  user_id: t.string,
+  lsns: jsonb(t.record(t.string)),
+  write_checkpoint: bigint
+});
+
+export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
+export type WriteCheckpointDecoded = t.Decoded<typeof WriteCheckpoint>;
+
+export const CustomWriteCheckpoint = t.object({
+  user_id: t.string,
+  write_checkpoint: bigint,
+  sync_rules_id: bigint
+});
+
+export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;
+export type CustomWriteCheckpointDecoded = t.Decoded<typeof CustomWriteCheckpoint>;

package/src/types/models/models-index.ts

@@ -0,0 +1,10 @@
+export * from './ActiveCheckpoint.js';
+export * from './ActiveCheckpointNotification.js';
+export * from './BucketData.js';
+export * from './BucketParameters.js';
+export * from './CurrentData.js';
+export * from './Instance.js';
+export * from './Migration.js';
+export * from './SourceTable.js';
+export * from './SyncRules.js';
+export * from './WriteCheckpoint.js';

package/src/types/types.ts

@@ -0,0 +1,73 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import * as pg_wire from '@powersync/service-jpgwire';
+import { configFile } from '@powersync/service-types';
+import * as t from 'ts-codec';
+export * as models from './models/models-index.js';
+
+export const MAX_BATCH_RECORD_COUNT = 2000;
+
+export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
+
+export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
+
+export const BatchLimits = t.object({
+  /**
+   * Maximum size of operations we write in a single transaction.
+   */
+  max_estimated_size: t.number.optional(),
+  /**
+   * Limit number of documents to write in a single transaction.
+   */
+  max_record_count: t.number.optional()
+});
+
+export type BatchLimits = t.Encoded<typeof BatchLimits>;
+
+export const OperationBatchLimits = BatchLimits.and(
+  t.object({
+    /**
+     * Maximum size of size of current_data documents we lookup at a time.
+     */
+    max_current_data_batch_size: t.number.optional()
+  })
+);
+
+export type OperationBatchLimits = t.Encoded<typeof OperationBatchLimits>;
+
+export const PostgresStorageConfig = configFile.BaseStorageConfig.and(lib_postgres.BasePostgresConnectionConfig).and(
+  t.object({
+    /**
+     * Allow batch operation limits to be configurable.
+     * Postgres has less batch size restrictions compared to MongoDB.
+     * Increasing limits can drastically improve replication performance, but
+     * can come at the cost of higher memory usage or potential issues.
+     */
+    batch_limits: OperationBatchLimits.optional()
+  })
+);
+
+export type PostgresStorageConfig = t.Encoded<typeof PostgresStorageConfig>;
+export type PostgresStorageConfigDecoded = t.Decoded<typeof PostgresStorageConfig>;
+
+export type RequiredOperationBatchLimits = Required<OperationBatchLimits>;
+
+export type NormalizedPostgresStorageConfig = pg_wire.NormalizedConnectionConfig & {
+  batch_limits: RequiredOperationBatchLimits;
+};
+
+export const normalizePostgresStorageConfig = (
+  baseConfig: PostgresStorageConfigDecoded
+): NormalizedPostgresStorageConfig => {
+  return {
+    ...lib_postgres.normalizeConnectionConfig(baseConfig),
+    batch_limits: {
+      max_current_data_batch_size: baseConfig.batch_limits?.max_current_data_batch_size ?? MAX_BATCH_CURRENT_DATA_SIZE,
+      max_estimated_size: baseConfig.batch_limits?.max_estimated_size ?? MAX_BATCH_ESTIMATED_SIZE,
+      max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT
+    }
+  };
+};
+
+export const isPostgresStorageConfig = (config: configFile.BaseStorageConfig): config is PostgresStorageConfig => {
+  return config.type == lib_postgres.POSTGRES_CONNECTION_TYPE;
+};

package/src/utils/bson.ts

@@ -0,0 +1,17 @@
+import { storage, utils } from '@powersync/service-core';
+import * as uuid from 'uuid';
+
+/**
+ * BSON is used to serialize certain documents for storage in BYTEA columns.
+ * JSONB columns do not directly support storing binary data which could be required in future.
+ */
+
+export function replicaIdToSubkey(tableId: string, id: storage.ReplicaId): string {
+  // Hashed UUID from the table and id
+  if (storage.isUUID(id)) {
+    // Special case for UUID for backwards-compatiblity
+    return `${tableId}/${id.toHexString()}`;
+  }
+  const repr = storage.serializeBson({ table: tableId, id });
+  return uuid.v5(repr, utils.ID_NAMESPACE);
+}

package/src/utils/bucket-data.ts

@@ -0,0 +1,25 @@
+import { utils } from '@powersync/service-core';
+import { models } from '../types/types.js';
+import { replicaIdToSubkey } from './bson.js';
+
+export const mapOpEntry = (entry: models.BucketDataDecoded) => {
+  if (entry.op == models.OpType.PUT || entry.op == models.OpType.REMOVE) {
+    return {
+      op_id: utils.timestampToOpId(entry.op_id),
+      op: entry.op,
+      object_type: entry.table_name ?? undefined,
+      object_id: entry.row_id ?? undefined,
+      checksum: Number(entry.checksum),
+      subkey: replicaIdToSubkey(entry.source_table!, entry.source_key!),
+      data: entry.data
+    };
+  } else {
+    // MOVE, CLEAR
+
+    return {
+      op_id: utils.timestampToOpId(entry.op_id),
+      op: entry.op,
+      checksum: Number(entry.checksum)
+    };
+  }
+};

package/src/utils/db.ts

@@ -0,0 +1,27 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+
+export const STORAGE_SCHEMA_NAME = 'powersync';
+
+export const NOTIFICATION_CHANNEL = 'powersynccheckpoints';
+
+/**
+ * Re export for prettier to detect the tag better
+ */
+export const sql = lib_postgres.sql;
+
+export const dropTables = async (client: lib_postgres.DatabaseClient) => {
+  // Lock a connection for automatic schema search paths
+  await client.lockConnection(async (db) => {
+    await db.sql`DROP TABLE IF EXISTS bucket_data`.execute();
+    await db.sql`DROP TABLE IF EXISTS bucket_parameters`.execute();
+    await db.sql`DROP TABLE IF EXISTS sync_rules`.execute();
+    await db.sql`DROP TABLE IF EXISTS instance`.execute();
+    await db.sql`DROP TABLE IF EXISTS bucket_data`.execute();
+    await db.sql`DROP TABLE IF EXISTS current_data`.execute();
+    await db.sql`DROP TABLE IF EXISTS source_tables`.execute();
+    await db.sql`DROP TABLE IF EXISTS write_checkpoints`.execute();
+    await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute();
+    await db.sql`DROP SEQUENCE IF EXISTS op_id_sequence`.execute();
+    await db.sql`DROP SEQUENCE IF EXISTS sync_rules_id_sequence`.execute();
+  });
+};

package/src/utils/ts-codec.ts

@@ -0,0 +1,14 @@
+import * as t from 'ts-codec';
+
+/**
+ * Returns a new codec with a subset of keys. Equivalent to the TypeScript Pick utility.
+ */
+export const pick = <T extends t.AnyObjectCodecShape, Keys extends keyof T>(codec: t.ObjectCodec<T>, keys: Keys[]) => {
+  // Filter the shape by the specified keys
+  const newShape = Object.fromEntries(
+    Object.entries(codec.props.shape).filter(([key]) => keys.includes(key as Keys))
+  ) as Pick<T, Keys>;
+
+  // Return a new codec with the narrowed shape
+  return t.object(newShape) as t.ObjectCodec<Pick<T, Keys>>;
+};

package/src/utils/utils-index.ts

@@ -0,0 +1,4 @@
+export * from './bson.js';
+export * from './bucket-data.js';
+export * from './db.js';
+export * from './ts-codec.js';

package/test/src/__snapshots__/storage.test.ts.snap

@@ -0,0 +1,9 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Postgres Sync Bucket Storage > empty storage metrics 1`] = `
+{
+  "operations_size_bytes": 16384,
+  "parameters_size_bytes": 32768,
+  "replication_size_bytes": 16384,
+}
+`;