@powersync/service-module-postgres-storage 0.11.2 → 0.12.0

package/src/storage/PostgresBucketStorageFactory.ts

@@ -1,7 +1,5 @@
-import * as framework from '@powersync/lib-services-framework';
-import { GetIntanceOptions, storage, SyncRulesBucketStorage, UpdateSyncRulesOptions } from '@powersync/service-core';
+import { GetIntanceOptions, storage, SyncRulesBucketStorage } from '@powersync/service-core';
 import * as pg_wire from '@powersync/service-jpgwire';
-import * as sync_rules from '@powersync/service-sync-rules';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
 
@@ -19,10 +17,7 @@ export type PostgresBucketStorageOptions = {
   slot_name_prefix: string;
 };
 
-export class PostgresBucketStorageFactory
-  extends framework.BaseObserver<storage.BucketStorageFactoryListener>
-  implements storage.BucketStorageFactory
-{
+export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
   readonly db: lib_postgres.DatabaseClient;
   public readonly slot_name_prefix: string;
 
@@ -145,42 +140,8 @@ export class PostgresBucketStorageFactory
     };
   }
 
-  // TODO possibly share implementation in abstract class
-  async configureSyncRules(options: UpdateSyncRulesOptions): Promise<{
-    updated: boolean;
-    persisted_sync_rules?: storage.PersistedSyncRulesContent;
-    lock?: storage.ReplicationLock;
-  }> {
-    const next = await this.getNextSyncRulesContent();
-    const active = await this.getActiveSyncRulesContent();
-
-    if (next?.sync_rules_content == options.content) {
-      framework.logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else if (next == null && active?.sync_rules_content == options.content) {
-      framework.logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else {
-      framework.logger.info('Sync rules updated from configuration');
-      const persisted_sync_rules = await this.updateSyncRules(options);
-      return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
-    }
-  }
-
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedSyncRulesContent> {
-    // TODO some shared implementation for this might be nice
-    if (options.validate) {
-      // Parse and validate before applying any changes
-      sync_rules.SqlSyncRules.fromYaml(options.content, {
-        // No schema-based validation at this point
-        schema: undefined,
-        defaultSchema: 'not_applicable', // Not needed for validation
-        throwOnError: true
-      });
-    } else {
-      // Apply unconditionally. Any errors will be reported via the diagnostics API.
-    }
-
+    const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
     return this.db.transaction(async (db) => {
       await db.sql`
         UPDATE sync_rules
@@ -197,7 +158,14 @@ export class PostgresBucketStorageFactory
               nextval('sync_rules_id_sequence') AS id
           )
         INSERT INTO
-          sync_rules (id, content, state, slot_name)
+          sync_rules (
+            id,
+            content,
+            sync_plan,
+            state,
+            slot_name,
+            storage_version
+          )
         VALUES
           (
             (
@@ -206,7 +174,8 @@ export class PostgresBucketStorageFactory
               FROM
                 next_id
             ),
-            ${{ type: 'varchar', value: options.content }},
+            ${{ type: 'varchar', value: options.config.yaml }},
+            ${{ type: 'json', value: options.config.plan }},
             ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
             CONCAT(
               ${{ type: 'varchar', value: this.slot_name_prefix }},
@@ -218,7 +187,8 @@ export class PostgresBucketStorageFactory
               ),
               '_',
               ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
-            )
+            ),
+            ${{ type: 'int4', value: storageVersion }}
           )
         RETURNING
           *
@@ -240,10 +210,8 @@ export class PostgresBucketStorageFactory
     // The current one will continue serving sync requests until the next one has finished processing.
     if (next != null && next.id == sync_rules_group_id) {
       // We need to redo the "next" sync rules
-      await this.updateSyncRules({
-        content: next.sync_rules_content,
-        validate: false
-      });
+
+      await this.updateSyncRules(next.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sql`
         UPDATE sync_rules
@@ -255,10 +223,7 @@ export class PostgresBucketStorageFactory
       `.execute();
     } else if (next == null && active?.id == sync_rules_group_id) {
       // Slot removed for "active" sync rules, while there is no "next" one.
-      await this.updateSyncRules({
-        content: active.sync_rules_content,
-        validate: false
-      });
+      await this.updateSyncRules(active.asUpdateOptions());
 
       // Pro-actively stop replicating, but still serve clients with existing data
       await this.db.sql`
@@ -284,12 +249,6 @@ export class PostgresBucketStorageFactory
     }
   }
 
-  // TODO possibly share via abstract class
-  async getActiveSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getActiveSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getActiveSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null> {
     const activeRow = await this.db.sql`
       SELECT
@@ -313,12 +272,6 @@ export class PostgresBucketStorageFactory
     return new PostgresPersistedSyncRulesContent(this.db, activeRow);
   }
 
-  // TODO possibly share via abstract class
-  async getNextSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getNextSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getNextSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null> {
     const nextRow = await this.db.sql`
       SELECT

package/src/storage/PostgresCompactor.ts

@@ -75,37 +75,54 @@ export class PostgresCompactor {
   async compact() {
     if (this.buckets) {
       for (let bucket of this.buckets) {
-        // We can make this more efficient later on by iterating
-        // through the buckets in a single query.
-        // That makes batching more tricky, so we leave for later.
-        await this.compactInternal(bucket);
+        await this.compactSingleBucket(bucket);
       }
     } else {
-      await this.compactInternal(undefined);
+      await this.compactAllBuckets();
     }
   }
 
-  async compactInternal(bucket: string | undefined) {
-    const idLimitBytes = this.idLimitBytes;
+  private async compactAllBuckets() {
+    const DISCOVERY_BATCH_SIZE = 200;
+    let lastBucket = '';
+
+    while (true) {
+      const bucketRows = (await this.db.sql`
+        SELECT DISTINCT
+          bucket_name
+        FROM
+          bucket_data
+        WHERE
+          group_id = ${{ type: 'int4', value: this.group_id }}
+          AND bucket_name > ${{ type: 'varchar', value: lastBucket }}
+        ORDER BY
+          bucket_name ASC
+        LIMIT
+          ${{ type: 'int4', value: DISCOVERY_BATCH_SIZE }}
+      `.rows()) as { bucket_name: string }[];
+
+      if (bucketRows.length === 0) {
+        break;
+      }
+
+      for (const row of bucketRows) {
+        await this.compactSingleBucket(row.bucket_name);
+      }
 
-    let currentState: CurrentBucketState | null = null;
-
-    let bucketLower: string | null = null;
-    let bucketUpper: string | null = null;
-    const MAX_CHAR = String.fromCodePoint(0xffff);
-
-    if (bucket == null) {
-      bucketLower = '';
-      bucketUpper = MAX_CHAR;
-    } else if (bucket?.includes('[')) {
-      // Exact bucket name
-      bucketLower = bucket;
-      bucketUpper = bucket;
-    } else if (bucket) {
-      // Bucket definition name
-      bucketLower = `${bucket}[`;
-      bucketUpper = `${bucket}[${MAX_CHAR}`;
+      lastBucket = bucketRows[bucketRows.length - 1].bucket_name;
     }
+  }
+
+  private async compactSingleBucket(bucket: string) {
+    const idLimitBytes = this.idLimitBytes;
+
+    let currentState: CurrentBucketState = {
+      bucket: bucket,
+      seen: new Map(),
+      trackingSize: 0,
+      lastNotPut: null,
+      opsSincePut: 0
+    };
 
     let upperOpIdLimit = BIGINT_MAX;
 
@@ -123,16 +140,9 @@ export class PostgresCompactor {
           bucket_data
         WHERE
           group_id = ${{ type: 'int4', value: this.group_id }}
-          AND bucket_name >= ${{ type: 'varchar', value: bucketLower }}
-          AND (
-            (
-              bucket_name = ${{ type: 'varchar', value: bucketUpper }}
-              AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
-            )
-            OR bucket_name < ${{ type: 'varchar', value: bucketUpper }} COLLATE "C" -- Use binary comparison
-          )
+          AND bucket_name = ${{ type: 'varchar', value: bucket }}
+          AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
         ORDER BY
-          bucket_name DESC,
           op_id DESC
         LIMIT
           ${{ type: 'int4', value: this.moveBatchQueryLimit }}
@@ -150,32 +160,8 @@ export class PostgresCompactor {
       // Set upperBound for the next batch
       const lastBatchItem = batch[batch.length - 1];
       upperOpIdLimit = lastBatchItem.op_id;
-      bucketUpper = lastBatchItem.bucket_name;
 
       for (const doc of batch) {
-        if (currentState == null || doc.bucket_name != currentState.bucket) {
-          if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
-            // Important to flush before clearBucket()
-            await this.flush();
-            logger.info(
-              `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
-            );
-
-            const bucket = currentState.bucket;
-            const clearOp = currentState.lastNotPut;
-            // Free memory before clearing bucket
-            currentState = null;
-            await this.clearBucket(bucket, clearOp);
-          }
-          currentState = {
-            bucket: doc.bucket_name,
-            seen: new Map(),
-            trackingSize: 0,
-            lastNotPut: null,
-            opsSincePut: 0
-          };
-        }
-
         if (this.maxOpId != null && doc.op_id > this.maxOpId) {
           continue;
         }
@@ -237,16 +223,12 @@ export class PostgresCompactor {
     }
 
     await this.flush();
-    currentState?.seen.clear();
-    if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) {
+    currentState.seen.clear();
+    if (currentState.lastNotPut != null && currentState.opsSincePut > 1) {
       logger.info(
         `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
       );
-      const bucket = currentState.bucket;
-      const clearOp = currentState.lastNotPut;
-      // Free memory before clearing bucket
-      currentState = null;
-      await this.clearBucket(bucket, clearOp);
+      await this.clearBucket(currentState.bucket, currentState.lastNotPut);
     }
   }
 

package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts

@@ -1,47 +1,27 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
-import { SqlSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
-
 import { models } from '../../types/types.js';
 
-export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncRulesContent {
-  public readonly slot_name: string;
-
-  public readonly id: number;
-  public readonly sync_rules_content: string;
-  public readonly last_checkpoint_lsn: string | null;
-  public readonly last_fatal_error: string | null;
-  public readonly last_keepalive_ts: Date | null;
-  public readonly last_checkpoint_ts: Date | null;
-  public readonly active: boolean;
+export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
   current_lock: storage.ReplicationLock | null = null;
 
   constructor(
     private db: lib_postgres.DatabaseClient,
     row: models.SyncRulesDecoded
   ) {
-    this.id = Number(row.id);
-    this.sync_rules_content = row.content;
-    this.last_checkpoint_lsn = row.last_checkpoint_lsn;
-    this.slot_name = row.slot_name;
-    this.last_fatal_error = row.last_fatal_error;
-    this.last_checkpoint_ts = row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null;
-    this.last_keepalive_ts = row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null;
-    this.active = row.state == 'ACTIVE';
-  }
-
-  parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules {
-    return {
-      id: this.id,
-      slot_name: this.slot_name,
-      sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options),
-      hydratedSyncRules() {
-        return this.sync_rules.config.hydrate({
-          hydrationState: versionedHydrationState(this.id)
-        });
-      }
-    };
+    super({
+      id: Number(row.id),
+      sync_rules_content: row.content,
+      compiled_plan: row.sync_plan,
+      last_checkpoint_lsn: row.last_checkpoint_lsn,
+      slot_name: row.slot_name,
+      last_fatal_error: row.last_fatal_error,
+      last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null,
+      last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
+      active: row.state == 'ACTIVE',
+      storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
+    });
   }
 
   async lock(): Promise<storage.ReplicationLock> {

package/src/types/models/SyncRules.ts

@@ -1,6 +1,7 @@
 import { framework, storage } from '@powersync/service-core';
 import * as t from 'ts-codec';
 import { bigint, pgwire_number } from '../codecs.js';
+import { jsonContainerObject } from './json.js';
 
 export const SyncRules = t.object({
   id: pgwire_number,
@@ -47,7 +48,21 @@ export const SyncRules = t.object({
    */
   last_fatal_error: t.Null.or(t.string),
   keepalive_op: t.Null.or(bigint),
-  content: t.string
+  storage_version: t.Null.or(pgwire_number).optional(),
+  content: t.string,
+  sync_plan: t.Null.or(
+    jsonContainerObject(
+      t.object({
+        plan: t.any,
+        compatibility: t.object({
+          edition: t.number,
+          overrides: t.record(t.boolean),
+          maxTimeValuePrecision: t.number.optional()
+        }),
+        eventDescriptors: t.record(t.array(t.string))
+      })
+    )
+  )
 });
 
 export type SyncRules = t.Encoded<typeof SyncRules>;

package/src/types/models/json.ts

@@ -0,0 +1,26 @@
+import { JsonContainer } from '@powersync/service-jsonbig';
+import { Codec, codec } from 'ts-codec';
+
+/**
+ * Wraps a codec to support {@link JsonContainer} values.
+ *
+ * Because our postgres client implementation wraps JSON objects in a {@link JsonContainer}, this intermediate layer is
+ * required to use JSON columns from Postgres in `ts-codec` models.
+ *
+ * Note that this serializes and deserializes values using {@link JSON}, so bigints are not supported.
+ */
+export function jsonContainerObject<I, O>(inner: Codec<I, O>): Codec<I, JsonContainer> {
+  return codec(
+    inner._tag,
+    (input) => {
+      return new JsonContainer(JSON.stringify(inner.encode(input)));
+    },
+    (json) => {
+      if (!(json instanceof JsonContainer)) {
+        throw new Error('Expected JsonContainer');
+      }
+
+      return inner.decode(JSON.parse(json.data));
+    }
+  );
+}

package/src/utils/db.ts

@@ -9,6 +9,9 @@ export const NOTIFICATION_CHANNEL = 'powersynccheckpoints';
  */
 export const sql = lib_postgres.sql;
 
+/**
+ * Drop all Postgres storage tables used by the service, including migrations.
+ */
 export const dropTables = async (client: lib_postgres.DatabaseClient) => {
   // Lock a connection for automatic schema search paths
   await client.lockConnection(async (db) => {
@@ -23,5 +26,39 @@ export const dropTables = async (client: lib_postgres.DatabaseClient) => {
     await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute();
     await db.sql`DROP SEQUENCE IF EXISTS op_id_sequence`.execute();
     await db.sql`DROP SEQUENCE IF EXISTS sync_rules_id_sequence`.execute();
+    await db.sql`DROP TABLE IF EXISTS migrations`.execute();
   });
 };
+
+/**
+ * Clear all Postgres storage tables and reset sequences.
+ *
+ * Does not clear migration state.
+ */
+export const truncateTables = async (db: lib_postgres.DatabaseClient) => {
+  // Lock a connection for automatic schema search paths
+  await db.query(
+    {
+      statement: `TRUNCATE TABLE bucket_data,
+        bucket_parameters,
+        sync_rules,
+        instance,
+        current_data,
+        source_tables,
+        write_checkpoints,
+        custom_write_checkpoints,
+        connection_report_events RESTART IDENTITY CASCADE
+    `
+    },
+    {
+      statement: `ALTER SEQUENCE IF EXISTS op_id_sequence RESTART
+      WITH
+        1`
+    },
+    {
+      statement: `ALTER SEQUENCE IF EXISTS sync_rules_id_sequence RESTART
+      WITH
+        1`
+    }
+  );
+};

package/src/utils/test-utils.ts

@@ -3,6 +3,7 @@ import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'
 import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
 import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
 import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
+import { truncateTables } from './db.js';
 
 export type PostgresTestStorageOptions = {
   url: string;
@@ -22,7 +23,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
   const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG);
 
-  const migrate = async (direction: framework.migrations.Direction) => {
+  const runMigrations = async (options: { down: boolean; up: boolean }) => {
     await using migrationManager: PowerSyncMigrationManager = new framework.MigrationManager();
     await using migrationAgent = factoryOptions.migrationAgent
       ? factoryOptions.migrationAgent(BASE_CONFIG)
@@ -31,14 +32,16 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
     const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext;
 
-    await migrationManager.migrate({
-      direction: framework.migrations.Direction.Down,
-      migrationContext: {
-        service_context: mockServiceContext
-      }
-    });
+    if (options.down) {
+      await migrationManager.migrate({
+        direction: framework.migrations.Direction.Down,
+        migrationContext: {
+          service_context: mockServiceContext
+        }
+      });
+    }
 
-    if (direction == framework.migrations.Direction.Up) {
+    if (options.up) {
       await migrationManager.migrate({
         direction: framework.migrations.Direction.Up,
         migrationContext: {
@@ -48,11 +51,28 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
     }
   };
 
+  const migrate = async (direction: framework.migrations.Direction) => {
+    await runMigrations({
+      down: true,
+      up: direction == framework.migrations.Direction.Up
+    });
+  };
+
+  const clearStorage = async () => {
+    await runMigrations({ down: false, up: true });
+
+    await using storageFactory = new PostgresBucketStorageFactory({
+      config: TEST_CONNECTION_OPTIONS,
+      slot_name_prefix: 'test_'
+    });
+    await truncateTables(storageFactory.db);
+  };
+
   return {
     reportFactory: async (options?: TestStorageOptions) => {
       try {
         if (!options?.doNotClear) {
-          await migrate(framework.migrations.Direction.Up);
+          await clearStorage();
         }
 
         return new PostgresReportStorage({
@@ -67,7 +87,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
     factory: async (options?: TestStorageOptions) => {
       try {
         if (!options?.doNotClear) {
-          await migrate(framework.migrations.Direction.Up);
+          await clearStorage();
         }
 
         return new PostgresBucketStorageFactory({