@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts

@@ -32,32 +32,172 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     return batchCreateCustomWriteCheckpoints(this.db, checkpoints);
   }
 
-  async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
+  async createManagedWriteCheckpoints(
+    checkpoints: storage.ManagedWriteCheckpointOptions[]
+  ): Promise<storage.CreateManagedWriteCheckpointsResult> {
     if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
       throw new framework.errors.ValidationError(
         `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
       );
     }
 
-    const row = await this.db.sql`
-      INSERT INTO
-        write_checkpoints (user_id, lsns, write_checkpoint)
-      VALUES
-        (
-          ${{ type: 'varchar', value: checkpoint.user_id }},
-          ${{ type: 'jsonb', value: checkpoint.heads }},
-          ${{ type: 'int8', value: 1 }}
-        )
-      ON CONFLICT (user_id) DO UPDATE
-      SET
-        write_checkpoint = write_checkpoints.write_checkpoint + 1,
-        lsns = EXCLUDED.lsns
-      RETURNING
-        *;
-    `
-      .decoded(models.WriteCheckpoint)
-      .first();
-    return row!.write_checkpoint;
+    const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints);
+    if (uniqueCheckpoints.length == 0) {
+      return { writeCheckpoints: new Map(), shouldAdvance: false };
+    }
+
+    const writeCheckpoints = new Map<string, bigint>();
+    const generatedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id == null);
+    const suppliedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id != null);
+
+    if (generatedCheckpoints.length > 0) {
+      const mappedCheckpoints = generatedCheckpoints.map((checkpoint) => ({
+        user_id: checkpoint.user_id,
+        lsns: checkpoint.heads
+      }));
+
+      const generatedRows = await this.db.sql`
+        WITH
+          json_data AS (
+            SELECT
+            CHECKPOINT ->> 'user_id' AS user_id,
+            CHECKPOINT -> 'lsns' AS lsns
+            FROM
+              jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
+            CHECKPOINT
+          )
+        INSERT INTO
+          write_checkpoints (
+            user_id,
+            lsns,
+            write_checkpoint,
+            checkpoint_requested_at
+          )
+        SELECT
+          user_id,
+          lsns,
+          1,
+          NULL
+        FROM
+          json_data
+        ON CONFLICT (user_id) DO UPDATE
+        SET
+          write_checkpoint = write_checkpoints.write_checkpoint + 1,
+          lsns = EXCLUDED.lsns,
+          checkpoint_requested_at = NULL
+        RETURNING
+          *;
+      `
+        .decoded(models.WriteCheckpoint)
+        .rows();
+
+      for (const row of generatedRows) {
+        writeCheckpoints.set(row.user_id, row.write_checkpoint);
+      }
+    }
+
+    if (suppliedCheckpoints.length > 0) {
+      // Supplied request ids are monotonic. Greater values update the checkpoint
+      // id and heads, while equal retries only refresh the retention timestamp.
+      // Stale requests return the stored id without changing the row.
+      const mappedCheckpoints = suppliedCheckpoints.map((checkpoint) => ({
+        user_id: checkpoint.user_id,
+        lsns: checkpoint.heads,
+        checkpoint_request_id: String(checkpoint.checkpoint_request_id)
+      }));
+
+      const suppliedRows = await this.db.sql`
+        WITH
+          json_data AS (
+            SELECT
+            CHECKPOINT ->> 'user_id' AS user_id,
+            CHECKPOINT -> 'lsns' AS lsns,
+            (
+              CHECKPOINT ->> 'checkpoint_request_id'
+            )::int8 AS checkpoint_request_id
+            FROM
+              jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
+            CHECKPOINT
+          )
+        INSERT INTO
+          write_checkpoints (
+            user_id,
+            lsns,
+            write_checkpoint,
+            checkpoint_requested_at
+          )
+        SELECT
+          user_id,
+          lsns,
+          checkpoint_request_id,
+          NOW()
+        FROM
+          json_data
+        ON CONFLICT (user_id) DO UPDATE
+        SET
+          write_checkpoint = CASE
+            WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.write_checkpoint
+            ELSE write_checkpoints.write_checkpoint
+          END,
+          lsns = CASE
+            WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.lsns
+            ELSE write_checkpoints.lsns
+          END,
+          checkpoint_requested_at = NOW()
+        WHERE
+          EXCLUDED.write_checkpoint >= write_checkpoints.write_checkpoint
+        RETURNING
+          *;
+      `
+        .decoded(models.WriteCheckpoint)
+        .rows();
+
+      for (const row of suppliedRows) {
+        writeCheckpoints.set(row.user_id, row.write_checkpoint);
+      }
+
+      // RETURNING only excludes stale requests rejected by the monotonic
+      // conflict condition. Those requests still need the stored id.
+      const returnedSuppliedUserIds = new Set(suppliedRows.map((row) => row.user_id));
+      const unchangedUserIds = suppliedCheckpoints
+        .map((checkpoint) => checkpoint.user_id)
+        .filter((userId) => !returnedSuppliedUserIds.has(userId));
+
+      if (unchangedUserIds.length > 0) {
+        const mappedUserIds = unchangedUserIds.map((user_id) => ({ user_id }));
+        const unchangedRows = await this.db.sql`
+          WITH
+            json_data AS (
+              SELECT
+              CHECKPOINT ->> 'user_id' AS user_id
+              FROM
+                jsonb_array_elements(${{ type: 'jsonb', value: mappedUserIds }}) AS
+              CHECKPOINT
+            )
+          SELECT
+            write_checkpoints.*
+          FROM
+            write_checkpoints
+            JOIN json_data ON write_checkpoints.user_id = json_data.user_id;
+        `
+          .decoded(models.WriteCheckpoint)
+          .rows();
+
+        for (const row of unchangedRows) {
+          writeCheckpoints.set(row.user_id, row.write_checkpoint);
+        }
+      }
+    }
+
+    // Postgres storage does not track a per-row processed indicator: a write
+    // checkpoint is considered processed at read time by comparing its lsns
+    // against the replicated head (see lastManagedWriteCheckpoint). We therefore
+    // force the source marker whenever any checkpoint was matched, which also
+    // covers stale or duplicate requests whose stored checkpoint may still be
+    // pending. Forcing a marker for an already-processed checkpoint is wasteful
+    // but harmless.
+    const shouldAdvance = generatedCheckpoints.length > 0 || suppliedCheckpoints.length > 0;
+    return { writeCheckpoints, shouldAdvance };
   }
 
   async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
@@ -133,7 +273,8 @@ export async function batchCreateCustomWriteCheckpoints(
       // Cannot encode bigint directly using JSON.stringify.
       // The ::int8 in the query below will take care of casting back to a number
       checkpoint: String(cp.checkpoint),
-      sync_rules_id: cp.sync_rules_id
+      sync_rules_id: cp.sync_rules_id,
+      checkpoint_requested_at: cp.checkpoint_requested_at?.toISOString() ?? null
     };
   });
 
@@ -145,7 +286,12 @@ export async function batchCreateCustomWriteCheckpoints(
         CHECKPOINT
       )
     INSERT INTO
-      custom_write_checkpoints (user_id, write_checkpoint, sync_rules_id)
+      custom_write_checkpoints (
+        user_id,
+        write_checkpoint,
+        sync_rules_id,
+        checkpoint_requested_at
+      )
     SELECT
     CHECKPOINT ->> 'user_id'::varchar,
     (
@@ -153,11 +299,15 @@ export async function batchCreateCustomWriteCheckpoints(
     )::int8,
     (
       CHECKPOINT ->> 'sync_rules_id'
-    )::int4
+    )::int4,
+    (
+      CHECKPOINT ->> 'checkpoint_requested_at'
+    )::timestamptz
     FROM
       json_data
     ON CONFLICT (user_id, sync_rules_id) DO UPDATE
     SET
-      write_checkpoint = EXCLUDED.write_checkpoint;
+      write_checkpoint = EXCLUDED.write_checkpoint,
+      checkpoint_requested_at = EXCLUDED.checkpoint_requested_at;
   `.execute();
 }

package/src/storage/current-data-store.ts

@@ -301,19 +301,74 @@ export class PostgresCurrentDataStore {
     `.execute();
   }
 
-  async deleteGroupRows(db: Queryable, options: { groupId: number }) {
+  /**
+   * Delete up to `limit` rows for the group, returning the number of candidate
+   * rows found by the scan (see PostgresSyncRulesStorage#deleteGroupBatch for
+   * why the scan is counted rather than the delete).
+   */
+  async deleteGroupRowsBatch(db: Queryable, options: { groupId: number; limit: number }): Promise<bigint> {
     if (this.softDeleteEnabled) {
-      await db.sql`
-        DELETE FROM v3_current_data
-        WHERE
-          group_id = ${{ type: 'int4', value: options.groupId }}
-      `.execute();
+      const result = await db.sql`
+        WITH
+          batch AS (
+            SELECT
+              ctid
+            FROM
+              v3_current_data
+            WHERE
+              group_id = ${{ type: 'int4', value: options.groupId }}
+            LIMIT
+              ${{ type: 'int4', value: options.limit }}
+          ),
+          deleted AS (
+            DELETE FROM v3_current_data
+            WHERE
+              ctid IN (
+                SELECT
+                  ctid
+                FROM
+                  batch
+              )
+            RETURNING
+              1
+          )
+        SELECT
+          COUNT(*) AS count
+        FROM
+          batch
+      `.first<{ count: bigint }>();
+      return result?.count ?? 0n;
     } else {
-      await db.sql`
-        DELETE FROM current_data
-        WHERE
-          group_id = ${{ type: 'int4', value: options.groupId }}
-      `.execute();
+      const result = await db.sql`
+        WITH
+          batch AS (
+            SELECT
+              ctid
+            FROM
+              current_data
+            WHERE
+              group_id = ${{ type: 'int4', value: options.groupId }}
+            LIMIT
+              ${{ type: 'int4', value: options.limit }}
+          ),
+          deleted AS (
+            DELETE FROM current_data
+            WHERE
+              ctid IN (
+                SELECT
+                  ctid
+                FROM
+                  batch
+              )
+            RETURNING
+              1
+          )
+        SELECT
+          COUNT(*) AS count
+        FROM
+          batch
+      `.first<{ count: bigint }>();
+      return result?.count ?? 0n;
     }
   }
 

package/src/storage/storage-index.ts

@@ -1,4 +1,5 @@
 export * from './PostgresBucketStorageFactory.js';
 export * from './PostgresCompactor.js';
+export * from './PostgresReportStorage.js';
 export * from './PostgresStorageProvider.js';
 export * from './PostgresSyncRulesStorage.js';

package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts

@@ -0,0 +1,98 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import { models } from '../../types/types.js';
+
+export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
+  constructor(
+    private db: lib_postgres.DatabaseClient,
+    row: models.SyncRulesDecoded
+  ) {
+    super({
+      replicationStreamId: Number(row.id),
+      sync_rules_content: row.content,
+      compiled_plan: row.sync_plan,
+      replicationStreamName: row.slot_name,
+      storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
+      syncConfigState: row.state as storage.SyncRuleState
+    });
+  }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    const row = await this.db.sql`
+      SELECT
+        *
+      FROM
+        sync_rules
+      WHERE
+        id = ${{ value: this.replicationStreamId, type: 'int4' }}
+    `
+      .decoded(models.SyncRules)
+      .first();
+
+    return row == null ? null : syncConfigStatusFromRow(row);
+  }
+}
+
+function syncConfigStatusFromRow(row: models.SyncRulesDecoded): storage.PersistedSyncConfigStatus {
+  return {
+    id: String(row.id),
+    replicationStreamId: Number(row.id),
+    state: row.state as storage.SyncRuleState,
+    snapshot_done: row.snapshot_done,
+    last_checkpoint_lsn: row.last_checkpoint_lsn,
+    last_fatal_error: row.last_fatal_error,
+    last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
+    last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
+  };
+}
+
+export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
+  current_lock: storage.ReplicationLock | null = null;
+  readonly syncConfigContent: readonly PostgresPersistedSyncConfigContent[];
+
+  constructor(
+    private db: lib_postgres.DatabaseClient,
+    private readonly row: models.SyncRulesDecoded
+  ) {
+    super({
+      replicationStreamId: Number(row.id),
+      replicationStreamName: row.slot_name,
+      state: row.state as storage.SyncRuleState,
+      storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
+    });
+    this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
+  }
+
+  parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
+    return this.syncConfigContent[0].parsed(options);
+  }
+
+  async lock(): Promise<storage.ReplicationLock> {
+    const manager = new lib_postgres.PostgresLockManager({
+      db: this.db,
+      name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
+    });
+    const lockHandle = await manager.acquire();
+    if (!lockHandle) {
+      throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
+    }
+
+    const interval = setInterval(async () => {
+      try {
+        await lockHandle.refresh();
+      } catch (e) {
+        this.logger.error('Failed to refresh lock', e);
+        clearInterval(interval);
+      }
+    }, 30_130);
+
+    return (this.current_lock = {
+      sync_rules_id: this.replicationStreamId,
+      release: async () => {
+        clearInterval(interval);
+        return lockHandle.release();
+      }
+    });
+  }
+}

package/src/types/models/WriteCheckpoint.ts

@@ -1,10 +1,12 @@
+import { framework } from '@powersync/service-core';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 
 export const WriteCheckpoint = t.object({
   user_id: t.string,
   lsns: jsonb(t.record(t.string)),
-  write_checkpoint: bigint
+  write_checkpoint: bigint,
+  checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 
 export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
@@ -13,7 +15,8 @@ export type WriteCheckpointDecoded = t.Decoded<typeof WriteCheckpoint>;
 export const CustomWriteCheckpoint = t.object({
   user_id: t.string,
   write_checkpoint: bigint,
-  sync_rules_id: bigint
+  sync_rules_id: bigint,
+  checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 
 export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;

package/src/utils/checkpoints.ts

@@ -0,0 +1,31 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { storage } from '@powersync/service-core';
+import { models } from '../types/types.js';
+
+/**
+ * Gets the latest active checkpoint document.
+ * This is mainly exported for mocking in tests.
+ */
+export async function getActiveCheckpointDocument({
+  db
+}: {
+  db: lib_postgres.DatabaseClient;
+}): Promise<models.ActiveCheckpointDecoded | null> {
+  return db.sql`
+    SELECT
+      id,
+      last_checkpoint,
+      last_checkpoint_lsn
+    FROM
+      sync_rules
+    WHERE
+      state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+      OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+    ORDER BY
+      id DESC
+    LIMIT
+      1
+  `
+    .decoded(models.ActiveCheckpoint)
+    .first();
+}

package/src/utils/test-utils.ts

@@ -73,7 +73,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
     await using storageFactory = new PostgresBucketStorageFactory({
       config: TEST_CONNECTION_OPTIONS,
-      slot_name_prefix: 'test_'
+      replicationStreamNamePrefix: 'test_'
     });
     await truncateTables(storageFactory.db);
   };
@@ -102,7 +102,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
         return new PostgresBucketStorageFactory({
           config: TEST_CONNECTION_OPTIONS,
-          slot_name_prefix: 'test_'
+          replicationStreamNamePrefix: 'test_'
         });
       } catch (ex) {
         // Vitest does not display these errors nicely when using the `await using` syntx