@powersync/service-module-postgres-storage 0.11.2 → 0.13.0

package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts

@@ -1,47 +1,27 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
-import { SqlSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
-
 import { models } from '../../types/types.js';
 
-export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncRulesContent {
-  public readonly slot_name: string;
-
-  public readonly id: number;
-  public readonly sync_rules_content: string;
-  public readonly last_checkpoint_lsn: string | null;
-  public readonly last_fatal_error: string | null;
-  public readonly last_keepalive_ts: Date | null;
-  public readonly last_checkpoint_ts: Date | null;
-  public readonly active: boolean;
+export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
   current_lock: storage.ReplicationLock | null = null;
 
   constructor(
     private db: lib_postgres.DatabaseClient,
     row: models.SyncRulesDecoded
   ) {
-    this.id = Number(row.id);
-    this.sync_rules_content = row.content;
-    this.last_checkpoint_lsn = row.last_checkpoint_lsn;
-    this.slot_name = row.slot_name;
-    this.last_fatal_error = row.last_fatal_error;
-    this.last_checkpoint_ts = row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null;
-    this.last_keepalive_ts = row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null;
-    this.active = row.state == 'ACTIVE';
-  }
-
-  parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules {
-    return {
-      id: this.id,
-      slot_name: this.slot_name,
-      sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options),
-      hydratedSyncRules() {
-        return this.sync_rules.config.hydrate({
-          hydrationState: versionedHydrationState(this.id)
-        });
-      }
-    };
+    super({
+      id: Number(row.id),
+      sync_rules_content: row.content,
+      compiled_plan: row.sync_plan,
+      last_checkpoint_lsn: row.last_checkpoint_lsn,
+      slot_name: row.slot_name,
+      last_fatal_error: row.last_fatal_error,
+      last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null,
+      last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
+      active: row.state == 'ACTIVE',
+      storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
+    });
   }
 
   async lock(): Promise<storage.ReplicationLock> {

package/src/storage/table-id.ts

@@ -0,0 +1,9 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+
+export function postgresTableId(id: storage.SourceTableId) {
+  if (typeof id == 'string') {
+    return id;
+  }
+  throw new ServiceAssertionError(`Expected string table id, got ObjectId`);
+}

package/src/types/models/CurrentData.ts

@@ -1,5 +1,5 @@
 import * as t from 'ts-codec';
-import { hexBuffer, jsonb, pgwire_number } from '../codecs.js';
+import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
 
 export const CurrentBucket = t.object({
   bucket: t.string,
@@ -10,7 +10,7 @@ export const CurrentBucket = t.object({
 export type CurrentBucket = t.Encoded<typeof CurrentBucket>;
 export type CurrentBucketDecoded = t.Decoded<typeof CurrentBucket>;
 
-export const CurrentData = t.object({
+export const V1CurrentData = t.object({
   buckets: jsonb(t.array(CurrentBucket)),
   data: hexBuffer,
   group_id: pgwire_number,
@@ -19,5 +19,18 @@ export const CurrentData = t.object({
   source_table: t.string
 });
 
-export type CurrentData = t.Encoded<typeof CurrentData>;
-export type CurrentDataDecoded = t.Decoded<typeof CurrentData>;
+export const V3CurrentData = t.object({
+  buckets: jsonb(t.array(CurrentBucket)),
+  data: hexBuffer,
+  group_id: pgwire_number,
+  lookups: t.array(hexBuffer),
+  source_key: hexBuffer,
+  source_table: t.string,
+  pending_delete: t.Null.or(bigint)
+});
+
+export type V1CurrentData = t.Encoded<typeof V1CurrentData>;
+export type V1CurrentDataDecoded = t.Decoded<typeof V1CurrentData>;
+
+export type V3CurrentData = t.Encoded<typeof V3CurrentData>;
+export type V3CurrentDataDecoded = t.Decoded<typeof V3CurrentData>;

package/src/types/models/SyncRules.ts

@@ -1,6 +1,7 @@
 import { framework, storage } from '@powersync/service-core';
 import * as t from 'ts-codec';
 import { bigint, pgwire_number } from '../codecs.js';
+import { jsonContainerObject } from './json.js';
 
 export const SyncRules = t.object({
   id: pgwire_number,
@@ -47,7 +48,21 @@ export const SyncRules = t.object({
    */
   last_fatal_error: t.Null.or(t.string),
   keepalive_op: t.Null.or(bigint),
-  content: t.string
+  storage_version: t.Null.or(pgwire_number).optional(),
+  content: t.string,
+  sync_plan: t.Null.or(
+    jsonContainerObject(
+      t.object({
+        plan: t.any,
+        compatibility: t.object({
+          edition: t.number,
+          overrides: t.record(t.boolean),
+          maxTimeValuePrecision: t.number.optional()
+        }),
+        eventDescriptors: t.record(t.array(t.string))
+      })
+    )
+  )
 });
 
 export type SyncRules = t.Encoded<typeof SyncRules>;

package/src/types/models/json.ts

@@ -0,0 +1,26 @@
+import { JsonContainer } from '@powersync/service-jsonbig';
+import { Codec, codec } from 'ts-codec';
+
+/**
+ * Wraps a codec to support {@link JsonContainer} values.
+ *
+ * Because our postgres client implementation wraps JSON objects in a {@link JsonContainer}, this intermediate layer is
+ * required to use JSON columns from Postgres in `ts-codec` models.
+ *
+ * Note that this serializes and deserializes values using {@link JSON}, so bigints are not supported.
+ */
+export function jsonContainerObject<I, O>(inner: Codec<I, O>): Codec<I, JsonContainer> {
+  return codec(
+    inner._tag,
+    (input) => {
+      return new JsonContainer(JSON.stringify(inner.encode(input)));
+    },
+    (json) => {
+      if (!(json instanceof JsonContainer)) {
+        throw new Error('Expected JsonContainer');
+      }
+
+      return inner.decode(JSON.parse(json.data));
+    }
+  );
+}

package/src/utils/bson.ts

@@ -6,7 +6,7 @@ import * as uuid from 'uuid';
  * JSONB columns do not directly support storing binary data which could be required in future.
  */
 
-export function replicaIdToSubkey(tableId: string, id: storage.ReplicaId): string {
+export function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string {
   // Hashed UUID from the table and id
   if (storage.isUUID(id)) {
     // Special case for UUID for backwards-compatiblity

package/src/utils/db.ts

@@ -9,6 +9,9 @@ export const NOTIFICATION_CHANNEL = 'powersynccheckpoints';
  */
 export const sql = lib_postgres.sql;
 
+/**
+ * Drop all Postgres storage tables used by the service, including migrations.
+ */
 export const dropTables = async (client: lib_postgres.DatabaseClient) => {
   // Lock a connection for automatic schema search paths
   await client.lockConnection(async (db) => {
@@ -18,10 +21,54 @@ export const dropTables = async (client: lib_postgres.DatabaseClient) => {
     await db.sql`DROP TABLE IF EXISTS instance`.execute();
     await db.sql`DROP TABLE IF EXISTS bucket_data`.execute();
     await db.sql`DROP TABLE IF EXISTS current_data`.execute();
+    await db.sql`DROP TABLE IF EXISTS v3_current_data`.execute();
     await db.sql`DROP TABLE IF EXISTS source_tables`.execute();
     await db.sql`DROP TABLE IF EXISTS write_checkpoints`.execute();
     await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute();
     await db.sql`DROP SEQUENCE IF EXISTS op_id_sequence`.execute();
     await db.sql`DROP SEQUENCE IF EXISTS sync_rules_id_sequence`.execute();
+    await db.sql`DROP TABLE IF EXISTS migrations`.execute();
   });
 };
+
+/**
+ * Clear all Postgres storage tables and reset sequences.
+ *
+ * Does not clear migration state.
+ */
+export const truncateTables = async (db: lib_postgres.DatabaseClient) => {
+  // Lock a connection for automatic schema search paths
+  await db.query(
+    {
+      statement: `TRUNCATE TABLE bucket_data,
+        bucket_parameters,
+        sync_rules,
+        instance,
+        current_data,
+        source_tables,
+        write_checkpoints,
+        custom_write_checkpoints,
+        connection_report_events RESTART IDENTITY CASCADE
+    `
+    },
+    {
+      // TRUNCATE if v3_current_data exists
+      statement: `DO $$
+        BEGIN
+          IF to_regclass('v3_current_data') IS NOT NULL THEN
+            EXECUTE 'TRUNCATE TABLE v3_current_data RESTART IDENTITY CASCADE';
+          END IF;
+        END $$;`
+    },
+    {
+      statement: `ALTER SEQUENCE IF EXISTS op_id_sequence RESTART
+      WITH
+        1`
+    },
+    {
+      statement: `ALTER SEQUENCE IF EXISTS sync_rules_id_sequence RESTART
+      WITH
+        1`
+    }
+  );
+};

package/src/utils/test-utils.ts

@@ -3,6 +3,8 @@ import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'
 import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
 import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
 import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
+import { logger as defaultLogger, createLogger, transports } from '@powersync/lib-services-framework';
+import { truncateTables } from './db.js';
 
 export type PostgresTestStorageOptions = {
   url: string;
@@ -22,7 +24,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
   const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG);
 
-  const migrate = async (direction: framework.migrations.Direction) => {
+  const runMigrations = async (options: { down: boolean; up: boolean }) => {
     await using migrationManager: PowerSyncMigrationManager = new framework.MigrationManager();
     await using migrationAgent = factoryOptions.migrationAgent
       ? factoryOptions.migrationAgent(BASE_CONFIG)
@@ -31,28 +33,56 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
     const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext;
 
-    await migrationManager.migrate({
-      direction: framework.migrations.Direction.Down,
-      migrationContext: {
-        service_context: mockServiceContext
-      }
+    // Migration logs can get really verbose in tests, so only log warnings and up.
+    const logger = createLogger({
+      level: 'warn',
+      format: defaultLogger.format,
+      transports: [new transports.Console()]
     });
 
-    if (direction == framework.migrations.Direction.Up) {
+    if (options.down) {
+      await migrationManager.migrate({
+        direction: framework.migrations.Direction.Down,
+        migrationContext: {
+          service_context: mockServiceContext
+        },
+        logger
+      });
+    }
+
+    if (options.up) {
       await migrationManager.migrate({
         direction: framework.migrations.Direction.Up,
         migrationContext: {
           service_context: mockServiceContext
-        }
+        },
+        logger
       });
     }
   };
 
+  const migrate = async (direction: framework.migrations.Direction) => {
+    await runMigrations({
+      down: true,
+      up: direction == framework.migrations.Direction.Up
+    });
+  };
+
+  const clearStorage = async () => {
+    await runMigrations({ down: false, up: true });
+
+    await using storageFactory = new PostgresBucketStorageFactory({
+      config: TEST_CONNECTION_OPTIONS,
+      slot_name_prefix: 'test_'
+    });
+    await truncateTables(storageFactory.db);
+  };
+
   return {
     reportFactory: async (options?: TestStorageOptions) => {
       try {
         if (!options?.doNotClear) {
-          await migrate(framework.migrations.Direction.Up);
+          await clearStorage();
         }
 
         return new PostgresReportStorage({
@@ -67,7 +97,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
     factory: async (options?: TestStorageOptions) => {
       try {
         if (!options?.doNotClear) {
-          await migrate(framework.migrations.Direction.Up);
+          await clearStorage();
         }
 
         return new PostgresBucketStorageFactory({
@@ -80,10 +110,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
         throw ex;
       }
     },
-    migrate
+    migrate,
+    tableIdStrings: true
   };
 }
-
-export function postgresTestStorageFactoryGenerator(factoryOptions: PostgresTestStorageOptions) {
-  return postgresTestSetup(factoryOptions).factory;
-}

package/test/src/__snapshots__/storage.test.ts.snap

@@ -1,9 +1,151 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`Postgres Sync Bucket Storage - Data > empty storage metrics 1`] = `
-{
-  "operations_size_bytes": 16384,
-  "parameters_size_bytes": 32768,
-  "replication_size_bytes": 16384,
-}
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 2`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
 `;

package/test/src/__snapshots__/storage_compacting.test.ts.snap

@@ -0,0 +1,17 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Postgres Sync Bucket Storage Compact > partial checksums after compacting (2) 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": 1196713877,
+  "count": 1,
+}
+`;
+
+exports[`Postgres Sync Bucket Storage Compact > partial checksums after compacting 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": -134691003,
+  "count": 4,
+}
+`;