@powersync/service-module-postgres-storage 0.0.0-dev-20250122110924 → 0.0.0-dev-20250227082606

package/src/storage/PostgresBucketStorageFactory.ts

@@ -1,11 +1,8 @@
 import * as framework from '@powersync/lib-services-framework';
-import { storage, sync, utils } from '@powersync/service-core';
+import { GetIntanceOptions, storage, SyncRulesBucketStorage, UpdateSyncRulesOptions } from '@powersync/service-core';
 import * as pg_wire from '@powersync/service-jpgwire';
 import * as sync_rules from '@powersync/service-sync-rules';
 import crypto from 'crypto';
-import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
-import { LRUCache } from 'lru-cache/min';
-import * as timers from 'timers/promises';
 import * as uuid from 'uuid';
 
 import * as lib_postgres from '@powersync/lib-service-postgres';
@@ -22,38 +19,13 @@ export type PostgresBucketStorageOptions = {
 };
 
 export class PostgresBucketStorageFactory
-  extends framework.DisposableObserver<storage.BucketStorageFactoryListener>
+  extends framework.BaseObserver<storage.BucketStorageFactoryListener>
   implements storage.BucketStorageFactory
 {
   readonly db: lib_postgres.DatabaseClient;
   public readonly slot_name_prefix: string;
 
-  private sharedIterator = new sync.BroadcastIterable((signal) => this.watchActiveCheckpoint(signal));
-
-  private readonly storageCache = new LRUCache<number, storage.SyncRulesBucketStorage>({
-    max: 3,
-    fetchMethod: async (id) => {
-      const syncRulesRow = await this.db.sql`
-        SELECT
-          *
-        FROM
-          sync_rules
-        WHERE
-          id = ${{ value: id, type: 'int4' }}
-      `
-        .decoded(models.SyncRules)
-        .first();
-      if (syncRulesRow == null) {
-        // Deleted in the meantime?
-        return undefined;
-      }
-      const rules = new PostgresPersistedSyncRulesContent(this.db, syncRulesRow);
-      return this.getInstance(rules);
-    },
-    dispose: (storage) => {
-      storage[Symbol.dispose]();
-    }
-  });
+  private activeStorageCache: storage.SyncRulesBucketStorage | undefined;
 
   constructor(protected options: PostgresBucketStorageOptions) {
     super();
@@ -70,7 +42,6 @@ export class PostgresBucketStorageFactory
   }
 
   async [Symbol.asyncDispose]() {
-    super[Symbol.dispose]();
     await this.db[Symbol.asyncDispose]();
   }
 
@@ -79,18 +50,22 @@ export class PostgresBucketStorageFactory
     // This has not been implemented yet.
   }
 
-  getInstance(syncRules: storage.PersistedSyncRulesContent): storage.SyncRulesBucketStorage {
+  getInstance(
+    syncRules: storage.PersistedSyncRulesContent,
+    options?: GetIntanceOptions
+  ): storage.SyncRulesBucketStorage {
     const storage = new PostgresSyncRulesStorage({
       factory: this,
       db: this.db,
       sync_rules: syncRules,
       batchLimits: this.options.config.batch_limits
     });
-    this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+    if (!options?.skipLifecycleHooks) {
+      this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+    }
     storage.registerListener({
       batchStarted: (batch) => {
-        // This nested listener will be automatically disposed when the storage is disposed
-        batch.registerManagedListener(storage, {
+        batch.registerListener({
           replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
         });
       }
@@ -157,11 +132,19 @@ export class PostgresBucketStorageFactory
     return newInstanceRow!.id;
   }
 
+  async getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier> {
+    const id = lib_postgres.utils.encodePostgresSystemIdentifier(
+      await lib_postgres.utils.queryPostgresSystemIdentifier(this.db.pool)
+    );
+
+    return {
+      id,
+      type: lib_postgres.POSTGRES_CONNECTION_TYPE
+    };
+  }
+
   // TODO possibly share implementation in abstract class
-  async configureSyncRules(
-    sync_rules: string,
-    options?: { lock?: boolean }
-  ): Promise<{
+  async configureSyncRules(options: UpdateSyncRulesOptions): Promise<{
     updated: boolean;
     persisted_sync_rules?: storage.PersistedSyncRulesContent;
     lock?: storage.ReplicationLock;
@@ -169,31 +152,32 @@ export class PostgresBucketStorageFactory
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();
 
-    if (next?.sync_rules_content == sync_rules) {
+    if (next?.sync_rules_content == options.content) {
       framework.logger.info('Sync rules from configuration unchanged');
       return { updated: false };
-    } else if (next == null && active?.sync_rules_content == sync_rules) {
+    } else if (next == null && active?.sync_rules_content == options.content) {
       framework.logger.info('Sync rules from configuration unchanged');
       return { updated: false };
     } else {
       framework.logger.info('Sync rules updated from configuration');
-      const persisted_sync_rules = await this.updateSyncRules({
-        content: sync_rules,
-        lock: options?.lock
-      });
+      const persisted_sync_rules = await this.updateSyncRules(options);
       return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
     }
   }
 
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedSyncRulesContent> {
     // TODO some shared implementation for this might be nice
-    // Parse and validate before applying any changes
-    sync_rules.SqlSyncRules.fromYaml(options.content, {
-      // No schema-based validation at this point
-      schema: undefined,
-      defaultSchema: 'not_applicable', // Not needed for validation
-      throwOnError: true
-    });
+    if (options.validate) {
+      // Parse and validate before applying any changes
+      sync_rules.SqlSyncRules.fromYaml(options.content, {
+        // No schema-based validation at this point
+        schema: undefined,
+        defaultSchema: 'not_applicable', // Not needed for validation
+        throwOnError: true
+      });
+    } else {
+      // Apply unconditionally. Any errors will be reported via the diagnostics API.
+    }
 
     return this.db.transaction(async (db) => {
       await db.sql`
@@ -246,16 +230,17 @@ export class PostgresBucketStorageFactory
     });
   }
 
-  async slotRemoved(slot_name: string): Promise<void> {
+  async restartReplication(sync_rules_group_id: number): Promise<void> {
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();
 
     // In both the below cases, we create a new sync rules instance.
-    // The current one will continue erroring until the next one has finished processing.
-    if (next != null && next.slot_name == slot_name) {
+    // The current one will continue serving sync requests until the next one has finished processing.
+    if (next != null && next.id == sync_rules_group_id) {
       // We need to redo the "next" sync rules
       await this.updateSyncRules({
-        content: next.sync_rules_content
+        content: next.sync_rules_content,
+        validate: false
       });
       // Pro-actively stop replicating
       await this.db.sql`
@@ -266,17 +251,30 @@ export class PostgresBucketStorageFactory
           id = ${{ value: next.id, type: 'int4' }}
           AND state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }}
       `.execute();
-    } else if (next == null && active?.slot_name == slot_name) {
+    } else if (next == null && active?.id == sync_rules_group_id) {
       // Slot removed for "active" sync rules, while there is no "next" one.
       await this.updateSyncRules({
-        content: active.sync_rules_content
+        content: active.sync_rules_content,
+        validate: false
       });
 
-      // Pro-actively stop replicating
+      // Pro-actively stop replicating, but still serve clients with existing data
       await this.db.sql`
         UPDATE sync_rules
         SET
-          state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }}
+          state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+        WHERE
+          id = ${{ value: active.id, type: 'int4' }}
+          AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+      `.execute();
+    } else if (next != null && active?.id == sync_rules_group_id) {
+      // Already have "next" sync rules - don't update any.
+
+      // Pro-actively stop replicating, but still serve clients with existing data
+      await this.db.sql`
+        UPDATE sync_rules
+        SET
+          state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
         WHERE
           id = ${{ value: active.id, type: 'int4' }}
           AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
@@ -298,6 +296,7 @@ export class PostgresBucketStorageFactory
         sync_rules
       WHERE
         state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+        OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
       ORDER BY
         id DESC
       LIMIT
@@ -371,126 +370,21 @@ export class PostgresBucketStorageFactory
     return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row));
   }
 
-  async getActiveCheckpoint(): Promise<storage.ActiveCheckpoint> {
-    const activeCheckpoint = await this.db.sql`
-      SELECT
-        id,
-        last_checkpoint,
-        last_checkpoint_lsn
-      FROM
-        sync_rules
-      WHERE
-        state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
-      ORDER BY
-        id DESC
-      LIMIT
-        1
-    `
-      .decoded(models.ActiveCheckpoint)
-      .first();
-
-    return this.makeActiveCheckpoint(activeCheckpoint);
-  }
-
-  async *watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable<storage.WriteCheckpoint> {
-    let lastCheckpoint: utils.OpId | null = null;
-    let lastWriteCheckpoint: bigint | null = null;
-
-    const iter = wrapWithAbort(this.sharedIterator, signal);
-    for await (const cp of iter) {
-      const { checkpoint, lsn } = cp;
-
-      // lsn changes are not important by itself.
-      // What is important is:
-      // 1. checkpoint (op_id) changes.
-      // 2. write checkpoint changes for the specific user
-      const bucketStorage = await cp.getBucketStorage();
-      if (!bucketStorage) {
-        continue;
-      }
-
-      const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
-
-      const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({
-        user_id,
-        heads: {
-          ...lsnFilters
-        }
-      });
-
-      if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
-        // No change - wait for next one
-        // In some cases, many LSNs may be produced in a short time.
-        // Add a delay to throttle the write checkpoint lookup a bit.
-        await timers.setTimeout(20 + 10 * Math.random());
-        continue;
-      }
-
-      lastWriteCheckpoint = currentWriteCheckpoint;
-      lastCheckpoint = checkpoint;
-
-      yield { base: cp, writeCheckpoint: currentWriteCheckpoint };
+  async getActiveStorage(): Promise<SyncRulesBucketStorage | null> {
+    const content = await this.getActiveSyncRulesContent();
+    if (content == null) {
+      return null;
     }
-  }
 
-  protected async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<storage.ActiveCheckpoint> {
-    const doc = await this.db.sql`
-      SELECT
-        id,
-        last_checkpoint,
-        last_checkpoint_lsn
-      FROM
-        sync_rules
-      WHERE
-        state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }}
-      LIMIT
-        1
-    `
-      .decoded(models.ActiveCheckpoint)
-      .first();
-
-    const sink = new sync.LastValueSink<string>(undefined);
-
-    const disposeListener = this.db.registerListener({
-      notification: (notification) => sink.next(notification.payload)
-    });
-
-    signal.addEventListener('aborted', async () => {
-      disposeListener();
-      sink.complete();
-    });
-
-    yield this.makeActiveCheckpoint(doc);
-
-    let lastOp: storage.ActiveCheckpoint | null = null;
-    for await (const payload of sink.withSignal(signal)) {
-      if (signal.aborted) {
-        return;
-      }
-
-      const notification = models.ActiveCheckpointNotification.decode(payload);
-      const activeCheckpoint = this.makeActiveCheckpoint(notification.active_checkpoint);
-
-      if (lastOp == null || activeCheckpoint.lsn != lastOp.lsn || activeCheckpoint.checkpoint != lastOp.checkpoint) {
-        lastOp = activeCheckpoint;
-        yield activeCheckpoint;
-      }
+    // It is important that this instance is cached.
+    // Not for the instance construction itself, but to ensure that internal caches on the instance
+    // are re-used properly.
+    if (this.activeStorageCache?.group_id == content.id) {
+      return this.activeStorageCache;
+    } else {
+      const instance = this.getInstance(content);
+      this.activeStorageCache = instance;
+      return instance;
     }
   }
-
-  private makeActiveCheckpoint(row: models.ActiveCheckpointDecoded | null) {
-    return {
-      checkpoint: utils.timestampToOpId(row?.last_checkpoint ?? 0n),
-      lsn: row?.last_checkpoint_lsn ?? null,
-      hasSyncRules() {
-        return row != null;
-      },
-      getBucketStorage: async () => {
-        if (row == null) {
-          return null;
-        }
-        return (await this.storageCache.fetch(Number(row.id))) ?? null;
-      }
-    } satisfies storage.ActiveCheckpoint;
-  }
 }

package/src/storage/PostgresCompactor.ts

@@ -35,14 +35,7 @@ interface CurrentBucketState {
 /**
  * Additional options, primarily for testing.
  */
-export interface PostgresCompactOptions extends storage.CompactOptions {
-  /** Minimum of 2 */
-  clearBatchLimit?: number;
-  /** Minimum of 1 */
-  moveBatchLimit?: number;
-  /** Minimum of 1 */
-  moveBatchQueryLimit?: number;
-}
+export interface PostgresCompactOptions extends storage.CompactOptions {}
 
 const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
@@ -99,15 +92,19 @@ export class PostgresCompactor {
 
     let bucketLower: string | null = null;
     let bucketUpper: string | null = null;
+    const MAX_CHAR = String.fromCodePoint(0xffff);
 
-    if (bucket?.includes('[')) {
+    if (bucket == null) {
+      bucketLower = '';
+      bucketUpper = MAX_CHAR;
+    } else if (bucket?.includes('[')) {
       // Exact bucket name
       bucketLower = bucket;
       bucketUpper = bucket;
     } else if (bucket) {
       // Bucket definition name
       bucketLower = `${bucket}[`;
-      bucketUpper = `${bucket}[\uFFFF`;
+      bucketUpper = `${bucket}[${MAX_CHAR}`;
     }
 
     let upperOpIdLimit = BIGINT_MAX;
@@ -126,10 +123,16 @@ export class PostgresCompactor {
           bucket_data
         WHERE
           group_id = ${{ type: 'int4', value: this.group_id }}
-          AND bucket_name LIKE COALESCE(${{ type: 'varchar', value: bucketLower }}, '%')
-          AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
+          AND bucket_name >= ${{ type: 'varchar', value: bucketLower }}
+          AND (
+            (
+              bucket_name = ${{ type: 'varchar', value: bucketUpper }}
+              AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
+            )
+            OR bucket_name < ${{ type: 'varchar', value: bucketUpper }} COLLATE "C" -- Use binary comparison
+          )
         ORDER BY
-          bucket_name,
+          bucket_name DESC,
           op_id DESC
         LIMIT
           ${{ type: 'int4', value: this.moveBatchQueryLimit }}
@@ -145,7 +148,9 @@ export class PostgresCompactor {
       }
 
       // Set upperBound for the next batch
-      upperOpIdLimit = batch[batch.length - 1].op_id;
+      const lastBatchItem = batch[batch.length - 1];
+      upperOpIdLimit = lastBatchItem.op_id;
+      bucketUpper = lastBatchItem.bucket_name;
 
       for (const doc of batch) {
         if (currentState == null || doc.bucket_name != currentState.bucket) {