@powersync/service-module-postgres-storage 0.0.0-dev-20250214100224 → 0.0.0-dev-20250227082606

package/src/storage/PostgresSyncRulesStorage.ts

@@ -1,6 +1,15 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
-import { DisposableObserver, ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { storage, utils } from '@powersync/service-core';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import {
+  BroadcastIterable,
+  CHECKPOINT_INVALIDATE_ALL,
+  CheckpointChanges,
+  GetCheckpointChangesOptions,
+  LastValueSink,
+  storage,
+  utils,
+  WatchWriteCheckpointOptions
+} from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as uuid from 'uuid';
@@ -8,14 +17,18 @@ import { BIGINT_MAX } from '../types/codecs.js';
 import { models, RequiredOperationBatchLimits } from '../types/types.js';
 import { replicaIdToSubkey } from '../utils/bson.js';
 import { mapOpEntry } from '../utils/bucket-data.js';
+import * as timers from 'timers/promises';
 
+import * as framework from '@powersync/lib-services-framework';
 import { StatementParam } from '@powersync/service-jpgwire';
-import { StoredRelationId } from '../types/models/SourceTable.js';
+import { SourceTableDecoded, StoredRelationId } from '../types/models/SourceTable.js';
 import { pick } from '../utils/ts-codec.js';
 import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js';
 import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js';
 import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
 import { PostgresCompactor } from './PostgresCompactor.js';
+import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import { Decoded } from 'ts-codec';
 
 export type PostgresSyncRulesStorageOptions = {
   factory: PostgresBucketStorageFactory;
@@ -26,7 +39,7 @@ export type PostgresSyncRulesStorageOptions = {
 };
 
 export class PostgresSyncRulesStorage
-  extends DisposableObserver<storage.SyncRulesBucketStorageListener>
+  extends framework.BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
 {
   public readonly group_id: number;
@@ -34,6 +47,8 @@ export class PostgresSyncRulesStorage
   public readonly slot_name: string;
   public readonly factory: PostgresBucketStorageFactory;
 
+  private sharedIterator = new BroadcastIterable((signal) => this.watchActiveCheckpoint(signal));
+
   protected db: lib_postgres.DatabaseClient;
   protected writeCheckpointAPI: PostgresWriteCheckpointAPI;
 
@@ -151,21 +166,39 @@ export class PostgresSyncRulesStorage
       type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
     }));
     return this.db.transaction(async (db) => {
-      let sourceTableRow = await db.sql`
-        SELECT
-          *
-        FROM
-          source_tables
-        WHERE
-          group_id = ${{ type: 'int4', value: group_id }}
-          AND connection_id = ${{ type: 'int4', value: connection_id }}
-          AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-          AND schema_name = ${{ type: 'varchar', value: schema }}
-          AND table_name = ${{ type: 'varchar', value: table }}
-          AND replica_id_columns = ${{ type: 'jsonb', value: columns }}
-      `
-        .decoded(models.SourceTable)
-        .first();
+      let sourceTableRow: SourceTableDecoded | null;
+      if (objectId != null) {
+        sourceTableRow = await db.sql`
+          SELECT
+            *
+          FROM
+            source_tables
+          WHERE
+            group_id = ${{ type: 'int4', value: group_id }}
+            AND connection_id = ${{ type: 'int4', value: connection_id }}
+            AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
+            AND schema_name = ${{ type: 'varchar', value: schema }}
+            AND table_name = ${{ type: 'varchar', value: table }}
+            AND replica_id_columns = ${{ type: 'jsonb', value: columns }}
+        `
+          .decoded(models.SourceTable)
+          .first();
+      } else {
+        sourceTableRow = await db.sql`
+          SELECT
+            *
+          FROM
+            source_tables
+          WHERE
+            group_id = ${{ type: 'int4', value: group_id }}
+            AND connection_id = ${{ type: 'int4', value: connection_id }}
+            AND schema_name = ${{ type: 'varchar', value: schema }}
+            AND table_name = ${{ type: 'varchar', value: table }}
+            AND replica_id_columns = ${{ type: 'jsonb', value: columns }}
+        `
+          .decoded(models.SourceTable)
+          .first();
+      }
 
       if (sourceTableRow == null) {
         const row = await db.sql`
@@ -184,7 +217,7 @@ export class PostgresSyncRulesStorage
               ${{ type: 'varchar', value: uuid.v4() }},
               ${{ type: 'int4', value: group_id }},
               ${{ type: 'int4', value: connection_id }},
-              --- The objectId can be string | number, we store it as jsonb value
+              --- The objectId can be string | number | undefined, we store it as jsonb value
               ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
               ${{ type: 'varchar', value: schema }},
               ${{ type: 'varchar', value: table }},
@@ -211,25 +244,47 @@ export class PostgresSyncRulesStorage
       sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable);
       sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable);
 
-      const truncatedTables = await db.sql`
-        SELECT
-          *
-        FROM
-          source_tables
-        WHERE
-          group_id = ${{ type: 'int4', value: group_id }}
-          AND connection_id = ${{ type: 'int4', value: connection_id }}
-          AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
-          AND (
-            relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-            OR (
+      let truncatedTables: SourceTableDecoded[] = [];
+      if (objectId != null) {
+        // relation_id present - check for renamed tables
+        truncatedTables = await db.sql`
+          SELECT
+            *
+          FROM
+            source_tables
+          WHERE
+            group_id = ${{ type: 'int4', value: group_id }}
+            AND connection_id = ${{ type: 'int4', value: connection_id }}
+            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
+            AND (
+              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
+              OR (
+                schema_name = ${{ type: 'varchar', value: schema }}
+                AND table_name = ${{ type: 'varchar', value: table }}
+              )
+            )
+        `
+          .decoded(models.SourceTable)
+          .rows();
+      } else {
+        // relation_id not present - only check for changed replica_id_columns
+        truncatedTables = await db.sql`
+          SELECT
+            *
+          FROM
+            source_tables
+          WHERE
+            group_id = ${{ type: 'int4', value: group_id }}
+            AND connection_id = ${{ type: 'int4', value: connection_id }}
+            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
+            AND (
               schema_name = ${{ type: 'varchar', value: schema }}
               AND table_name = ${{ type: 'varchar', value: table }}
             )
-          )
-      `
-        .decoded(models.SourceTable)
-        .rows();
+        `
+          .decoded(models.SourceTable)
+          .rows();
+      }
 
       return {
         table: sourceTable,
@@ -272,7 +327,7 @@ export class PostgresSyncRulesStorage
 
     const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null;
 
-    await using batch = new PostgresBucketBatch({
+    const batch = new PostgresBucketBatch({
       db: this.db,
       sync_rules: this.sync_rules.parsed(options).sync_rules,
       group_id: this.group_id,
@@ -602,7 +657,10 @@ export class PostgresSyncRulesStorage
         SET
           state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }}
         WHERE
-          state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }}
+          (
+            state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+            OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+          )
           AND id != ${{ type: 'int4', value: this.group_id }}
       `.execute();
     });
@@ -663,4 +721,139 @@ export class PostgresSyncRulesStorage
       })
     );
   }
+
+  async getActiveCheckpoint(): Promise<storage.ReplicationCheckpoint> {
+    const activeCheckpoint = await this.db.sql`
+      SELECT
+        id,
+        last_checkpoint,
+        last_checkpoint_lsn
+      FROM
+        sync_rules
+      WHERE
+        state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+        OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+      ORDER BY
+        id DESC
+      LIMIT
+        1
+    `
+      .decoded(models.ActiveCheckpoint)
+      .first();
+
+    return this.makeActiveCheckpoint(activeCheckpoint);
+  }
+
+  async *watchWriteCheckpoint(options: WatchWriteCheckpointOptions): AsyncIterable<storage.StorageCheckpointUpdate> {
+    let lastCheckpoint: utils.OpId | null = null;
+    let lastWriteCheckpoint: bigint | null = null;
+
+    const { signal, user_id } = options;
+
+    const iter = wrapWithAbort(this.sharedIterator, signal);
+    for await (const cp of iter) {
+      const { checkpoint, lsn } = cp;
+
+      // lsn changes are not important by itself.
+      // What is important is:
+      // 1. checkpoint (op_id) changes.
+      // 2. write checkpoint changes for the specific user
+      const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
+
+      const currentWriteCheckpoint = await this.lastWriteCheckpoint({
+        user_id,
+        heads: {
+          ...lsnFilters
+        }
+      });
+
+      if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
+        // No change - wait for next one
+        // In some cases, many LSNs may be produced in a short time.
+        // Add a delay to throttle the write checkpoint lookup a bit.
+        await timers.setTimeout(20 + 10 * Math.random());
+        continue;
+      }
+
+      lastWriteCheckpoint = currentWriteCheckpoint;
+      lastCheckpoint = checkpoint;
+
+      yield {
+        base: cp,
+        writeCheckpoint: currentWriteCheckpoint,
+        update: CHECKPOINT_INVALIDATE_ALL
+      };
+    }
+  }
+
+  protected async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<storage.ReplicationCheckpoint> {
+    const doc = await this.db.sql`
+      SELECT
+        id,
+        last_checkpoint,
+        last_checkpoint_lsn
+      FROM
+        sync_rules
+      WHERE
+        state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+        OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+      LIMIT
+        1
+    `
+      .decoded(models.ActiveCheckpoint)
+      .first();
+
+    if (doc == null) {
+      // Abort the connections - clients will have to retry later.
+      throw new framework.ServiceError(framework.ErrorCode.PSYNC_S2302, 'No active sync rules available');
+    }
+
+    const sink = new LastValueSink<string>(undefined);
+
+    const disposeListener = this.db.registerListener({
+      notification: (notification) => sink.next(notification.payload)
+    });
+
+    signal.addEventListener('aborted', async () => {
+      disposeListener();
+      sink.complete();
+    });
+
+    yield this.makeActiveCheckpoint(doc);
+
+    let lastOp: storage.ReplicationCheckpoint | null = null;
+    for await (const payload of sink.withSignal(signal)) {
+      if (signal.aborted) {
+        return;
+      }
+
+      const notification = models.ActiveCheckpointNotification.decode(payload);
+      if (notification.active_checkpoint == null) {
+        continue;
+      }
+      if (Number(notification.active_checkpoint.id) != doc.id) {
+        // Active sync rules changed - abort and restart the stream
+        break;
+      }
+
+      const activeCheckpoint = this.makeActiveCheckpoint(notification.active_checkpoint);
+
+      if (lastOp == null || activeCheckpoint.lsn != lastOp.lsn || activeCheckpoint.checkpoint != lastOp.checkpoint) {
+        lastOp = activeCheckpoint;
+        yield activeCheckpoint;
+      }
+    }
+  }
+
+  async getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<CheckpointChanges> {
+    // We do not track individual changes yet
+    return CHECKPOINT_INVALIDATE_ALL;
+  }
+
+  private makeActiveCheckpoint(row: models.ActiveCheckpointDecoded | null) {
+    return {
+      checkpoint: utils.timestampToOpId(row?.last_checkpoint ?? 0n),
+      lsn: row?.last_checkpoint_lsn ?? null
+    } satisfies storage.ReplicationCheckpoint;
+  }
 }

package/src/storage/batch/PostgresBucketBatch.ts

@@ -1,7 +1,7 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import {
+  BaseObserver,
   container,
-  DisposableObserver,
   ErrorCode,
   errors,
   logger,
@@ -51,7 +51,7 @@ type StatefulCheckpointDecoded = t.Decoded<typeof StatefulCheckpoint>;
 const MAX_ROW_SIZE = 15 * 1024 * 1024;
 
 export class PostgresBucketBatch
-  extends DisposableObserver<storage.BucketBatchStorageListener>
+  extends BaseObserver<storage.BucketBatchStorageListener>
   implements storage.BucketStorageBatch
 {
   public last_flushed_op: bigint | null = null;
@@ -87,6 +87,10 @@ export class PostgresBucketBatch
     return this.last_checkpoint_lsn;
   }
 
+  async [Symbol.asyncDispose]() {
+    super.clearListeners();
+  }
+
   async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
     // TODO maybe share with abstract class
     const { after, before, sourceTable, tag } = record;

package/src/types/models/SourceTable.ts

@@ -2,7 +2,7 @@ import * as t from 'ts-codec';
 import { bigint, jsonb, jsonb_raw, pgwire_number } from '../codecs.js';
 
 export type StoredRelationId = {
-  object_id: string | number;
+  object_id: string | number | undefined;
 };
 
 export const ColumnDescriptor = t.object({

package/src/types/types.ts

@@ -53,6 +53,7 @@ export type RequiredOperationBatchLimits = Required<OperationBatchLimits>;
 
 export type NormalizedPostgresStorageConfig = pg_wire.NormalizedConnectionConfig & {
   batch_limits: RequiredOperationBatchLimits;
+  max_pool_size: number;
 };
 
 export const normalizePostgresStorageConfig = (

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -9,6 +9,7 @@ exports[`sync - postgres > compacting data - invalidate checkpoint 1`] = `
           "bucket": "mybucket[]",
           "checksum": -93886621,
           "count": 2,
+          "priority": 3,
         },
       ],
       "last_op_id": "2",
@@ -44,6 +45,7 @@ exports[`sync - postgres > compacting data - invalidate checkpoint 2`] = `
           "bucket": "mybucket[]",
           "checksum": 499012468,
           "count": 4,
+          "priority": 3,
         },
       ],
       "write_checkpoint": undefined,
@@ -102,6 +104,7 @@ exports[`sync - postgres > expiring token 1`] = `
           "bucket": "mybucket[]",
           "checksum": 0,
           "count": 0,
+          "priority": 3,
         },
       ],
       "last_op_id": "0",
@@ -124,6 +127,136 @@ exports[`sync - postgres > expiring token 2`] = `
 ]
 `;
 
+exports[`sync - postgres > sends checkpoint complete line for empty checkpoint 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -1221282404,
+          "count": 1,
+          "priority": 3,
+        },
+      ],
+      "last_op_id": "1",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 3073684892,
+          "data": "{"id":"t1","description":"sync"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [],
+      "write_checkpoint": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > sync buckets in order 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "b0[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 2,
+        },
+        {
+          "bucket": "b1[]",
+          "checksum": -1382098757,
+          "count": 1,
+          "priority": 1,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "b1[]",
+      "data": [
+        {
+          "checksum": 2912868539n,
+          "data": "{"id":"earlier","description":"Test 2"}",
+          "object_id": "earlier",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 1,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "b0[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - postgres > sync global data 1`] = `
 [
   {
@@ -133,6 +266,7 @@ exports[`sync - postgres > sync global data 1`] = `
           "bucket": "mybucket[]",
           "checksum": -93886621,
           "count": 2,
+          "priority": 3,
         },
       ],
       "last_op_id": "2",
@@ -184,6 +318,7 @@ exports[`sync - postgres > sync legacy non-raw data 1`] = `
           "bucket": "mybucket[]",
           "checksum": -852817836,
           "count": 1,
+          "priority": 3,
         },
       ],
       "last_op_id": "1",
@@ -231,6 +366,7 @@ exports[`sync - postgres > sync updates to global data 1`] = `
           "bucket": "mybucket[]",
           "checksum": 0,
           "count": 0,
+          "priority": 3,
         },
       ],
       "last_op_id": "0",
@@ -256,6 +392,7 @@ exports[`sync - postgres > sync updates to global data 2`] = `
           "bucket": "mybucket[]",
           "checksum": 920318466,
           "count": 1,
+          "priority": 3,
         },
       ],
       "write_checkpoint": undefined,
@@ -299,6 +436,7 @@ exports[`sync - postgres > sync updates to global data 3`] = `
           "bucket": "mybucket[]",
           "checksum": -93886621,
           "count": 2,
+          "priority": 3,
         },
       ],
       "write_checkpoint": undefined,

package/test/src/storage.test.ts

@@ -26,7 +26,7 @@ describe('Postgres Sync Bucket Storage', () => {
           - SELECT id, description FROM "%"
     `
     );
-    using factory = await POSTGRES_STORAGE_FACTORY();
+    await using factory = await POSTGRES_STORAGE_FACTORY();
     const bucketStorage = factory.getInstance(sync_rules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {

package/test/src/storage_compacting.test.ts

@@ -2,4 +2,4 @@ import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
 
-describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY, {}));
+describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY));