@powersync/service-module-postgres-storage 0.16.3 → 0.18.0

package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts

@@ -34,52 +34,170 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
 
   async createManagedWriteCheckpoints(
     checkpoints: storage.ManagedWriteCheckpointOptions[]
-  ): Promise<Map<string, bigint>> {
+  ): Promise<storage.CreateManagedWriteCheckpointsResult> {
     if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
       throw new framework.errors.ValidationError(
         `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
       );
     }
 
-    const uniqueCheckpoints = [...new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint])).values()];
+    const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints);
     if (uniqueCheckpoints.length == 0) {
-      return new Map();
+      return { writeCheckpoints: new Map(), shouldAdvance: false };
     }
 
-    const mappedCheckpoints = uniqueCheckpoints.map((checkpoint) => ({
-      user_id: checkpoint.user_id,
-      lsns: checkpoint.heads
-    }));
+    const writeCheckpoints = new Map<string, bigint>();
+    const generatedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id == null);
+    const suppliedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id != null);
 
-    const rows = await this.db.sql`
-      WITH
-        json_data AS (
+    if (generatedCheckpoints.length > 0) {
+      const mappedCheckpoints = generatedCheckpoints.map((checkpoint) => ({
+        user_id: checkpoint.user_id,
+        lsns: checkpoint.heads
+      }));
+
+      const generatedRows = await this.db.sql`
+        WITH
+          json_data AS (
+            SELECT
+            CHECKPOINT ->> 'user_id' AS user_id,
+            CHECKPOINT -> 'lsns' AS lsns
+            FROM
+              jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
+            CHECKPOINT
+          )
+        INSERT INTO
+          write_checkpoints (
+            user_id,
+            lsns,
+            write_checkpoint,
+            checkpoint_requested_at
+          )
+        SELECT
+          user_id,
+          lsns,
+          1,
+          NULL
+        FROM
+          json_data
+        ON CONFLICT (user_id) DO UPDATE
+        SET
+          write_checkpoint = write_checkpoints.write_checkpoint + 1,
+          lsns = EXCLUDED.lsns,
+          checkpoint_requested_at = NULL
+        RETURNING
+          *;
+      `
+        .decoded(models.WriteCheckpoint)
+        .rows();
+
+      for (const row of generatedRows) {
+        writeCheckpoints.set(row.user_id, row.write_checkpoint);
+      }
+    }
+
+    if (suppliedCheckpoints.length > 0) {
+      // Supplied request ids are monotonic. Greater values update the checkpoint
+      // id and heads, while equal retries only refresh the retention timestamp.
+      // Stale requests return the stored id without changing the row.
+      const mappedCheckpoints = suppliedCheckpoints.map((checkpoint) => ({
+        user_id: checkpoint.user_id,
+        lsns: checkpoint.heads,
+        checkpoint_request_id: String(checkpoint.checkpoint_request_id)
+      }));
+
+      const suppliedRows = await this.db.sql`
+        WITH
+          json_data AS (
+            SELECT
+            CHECKPOINT ->> 'user_id' AS user_id,
+            CHECKPOINT -> 'lsns' AS lsns,
+            (
+              CHECKPOINT ->> 'checkpoint_request_id'
+            )::int8 AS checkpoint_request_id
+            FROM
+              jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
+            CHECKPOINT
+          )
+        INSERT INTO
+          write_checkpoints (
+            user_id,
+            lsns,
+            write_checkpoint,
+            checkpoint_requested_at
+          )
+        SELECT
+          user_id,
+          lsns,
+          checkpoint_request_id,
+          NOW()
+        FROM
+          json_data
+        ON CONFLICT (user_id) DO UPDATE
+        SET
+          write_checkpoint = CASE
+            WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.write_checkpoint
+            ELSE write_checkpoints.write_checkpoint
+          END,
+          lsns = CASE
+            WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.lsns
+            ELSE write_checkpoints.lsns
+          END,
+          checkpoint_requested_at = NOW()
+        WHERE
+          EXCLUDED.write_checkpoint >= write_checkpoints.write_checkpoint
+        RETURNING
+          *;
+      `
+        .decoded(models.WriteCheckpoint)
+        .rows();
+
+      for (const row of suppliedRows) {
+        writeCheckpoints.set(row.user_id, row.write_checkpoint);
+      }
+
+      // RETURNING only excludes stale requests rejected by the monotonic
+      // conflict condition. Those requests still need the stored id.
+      const returnedSuppliedUserIds = new Set(suppliedRows.map((row) => row.user_id));
+      const unchangedUserIds = suppliedCheckpoints
+        .map((checkpoint) => checkpoint.user_id)
+        .filter((userId) => !returnedSuppliedUserIds.has(userId));
+
+      if (unchangedUserIds.length > 0) {
+        const mappedUserIds = unchangedUserIds.map((user_id) => ({ user_id }));
+        const unchangedRows = await this.db.sql`
+          WITH
+            json_data AS (
+              SELECT
+              CHECKPOINT ->> 'user_id' AS user_id
+              FROM
+                jsonb_array_elements(${{ type: 'jsonb', value: mappedUserIds }}) AS
+              CHECKPOINT
+            )
           SELECT
-          CHECKPOINT ->> 'user_id' AS user_id,
-          CHECKPOINT -> 'lsns' AS lsns
+            write_checkpoints.*
           FROM
-            jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
-          CHECKPOINT
-        )
-      INSERT INTO
-        write_checkpoints (user_id, lsns, write_checkpoint)
-      SELECT
-        user_id,
-        lsns,
-        1
-      FROM
-        json_data
-      ON CONFLICT (user_id) DO UPDATE
-      SET
-        write_checkpoint = write_checkpoints.write_checkpoint + 1,
-        lsns = EXCLUDED.lsns
-      RETURNING
-        *;
-    `
-      .decoded(models.WriteCheckpoint)
-      .rows();
+            write_checkpoints
+            JOIN json_data ON write_checkpoints.user_id = json_data.user_id;
+        `
+          .decoded(models.WriteCheckpoint)
+          .rows();
+
+        for (const row of unchangedRows) {
+          writeCheckpoints.set(row.user_id, row.write_checkpoint);
+        }
+      }
+    }
 
-    return new Map(rows.map((row) => [row.user_id, row.write_checkpoint]));
+    // Postgres storage does not track a per-row processed indicator: a write
+    // checkpoint is considered processed at read time by comparing its lsns
+    // against the replicated head (see lastManagedWriteCheckpoint). We therefore
+    // force the source marker whenever any checkpoint was matched, which also
+    // covers stale or duplicate requests whose stored checkpoint may still be
+    // pending. Forcing a marker for an already-processed checkpoint is wasteful
+    // but harmless.
+    const shouldAdvance = generatedCheckpoints.length > 0 || suppliedCheckpoints.length > 0;
+    return { writeCheckpoints, shouldAdvance };
   }
 
   async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
@@ -155,7 +273,8 @@ export async function batchCreateCustomWriteCheckpoints(
       // Cannot encode bigint directly using JSON.stringify.
       // The ::int8 in the query below will take care of casting back to a number
       checkpoint: String(cp.checkpoint),
-      sync_rules_id: cp.sync_rules_id
+      sync_rules_id: cp.sync_rules_id,
+      checkpoint_requested_at: cp.checkpoint_requested_at?.toISOString() ?? null
     };
   });
 
@@ -167,7 +286,12 @@ export async function batchCreateCustomWriteCheckpoints(
         CHECKPOINT
       )
     INSERT INTO
-      custom_write_checkpoints (user_id, write_checkpoint, sync_rules_id)
+      custom_write_checkpoints (
+        user_id,
+        write_checkpoint,
+        sync_rules_id,
+        checkpoint_requested_at
+      )
     SELECT
     CHECKPOINT ->> 'user_id'::varchar,
     (
@@ -175,11 +299,15 @@ export async function batchCreateCustomWriteCheckpoints(
     )::int8,
     (
       CHECKPOINT ->> 'sync_rules_id'
-    )::int4
+    )::int4,
+    (
+      CHECKPOINT ->> 'checkpoint_requested_at'
+    )::timestamptz
     FROM
       json_data
     ON CONFLICT (user_id, sync_rules_id) DO UPDATE
     SET
-      write_checkpoint = EXCLUDED.write_checkpoint;
+      write_checkpoint = EXCLUDED.write_checkpoint,
+      checkpoint_requested_at = EXCLUDED.checkpoint_requested_at;
   `.execute();
 }

package/src/storage/current-data-store.ts

@@ -301,19 +301,74 @@ export class PostgresCurrentDataStore {
     `.execute();
   }
 
-  async deleteGroupRows(db: Queryable, options: { groupId: number }) {
+  /**
+   * Delete up to `limit` rows for the group, returning the number of candidate
+   * rows found by the scan (see PostgresSyncRulesStorage#deleteGroupBatch for
+   * why the scan is counted rather than the delete).
+   */
+  async deleteGroupRowsBatch(db: Queryable, options: { groupId: number; limit: number }): Promise<bigint> {
     if (this.softDeleteEnabled) {
-      await db.sql`
-        DELETE FROM v3_current_data
-        WHERE
-          group_id = ${{ type: 'int4', value: options.groupId }}
-      `.execute();
+      const result = await db.sql`
+        WITH
+          batch AS (
+            SELECT
+              ctid
+            FROM
+              v3_current_data
+            WHERE
+              group_id = ${{ type: 'int4', value: options.groupId }}
+            LIMIT
+              ${{ type: 'int4', value: options.limit }}
+          ),
+          deleted AS (
+            DELETE FROM v3_current_data
+            WHERE
+              ctid IN (
+                SELECT
+                  ctid
+                FROM
+                  batch
+              )
+            RETURNING
+              1
+          )
+        SELECT
+          COUNT(*) AS count
+        FROM
+          batch
+      `.first<{ count: bigint }>();
+      return result?.count ?? 0n;
     } else {
-      await db.sql`
-        DELETE FROM current_data
-        WHERE
-          group_id = ${{ type: 'int4', value: options.groupId }}
-      `.execute();
+      const result = await db.sql`
+        WITH
+          batch AS (
+            SELECT
+              ctid
+            FROM
+              current_data
+            WHERE
+              group_id = ${{ type: 'int4', value: options.groupId }}
+            LIMIT
+              ${{ type: 'int4', value: options.limit }}
+          ),
+          deleted AS (
+            DELETE FROM current_data
+            WHERE
+              ctid IN (
+                SELECT
+                  ctid
+                FROM
+                  batch
+              )
+            RETURNING
+              1
+          )
+        SELECT
+          COUNT(*) AS count
+        FROM
+          batch
+      `.first<{ count: bigint }>();
+      return result?.count ?? 0n;
     }
   }
 

package/src/types/models/SourceTable.ts

@@ -1,3 +1,4 @@
+import { JsonValue } from '@powersync/service-core';
 import * as t from 'ts-codec';
 import { bigint, hexBuffer, jsonb, jsonb_raw, pgwire_number } from '../codecs.js';
 
@@ -14,7 +15,7 @@ export const ColumnDescriptor = t.object({
   /**
    *  Some data sources have a type id that can be used to identify the type of the column
    */
-  typeId: t.number.optional()
+  type_oid: t.number.optional()
 });
 
 export const SourceTable = t.object({
@@ -28,7 +29,11 @@ export const SourceTable = t.object({
   snapshot_done: t.boolean,
   snapshot_total_estimated_count: t.Null.or(bigint),
   snapshot_replicated_count: t.Null.or(bigint),
-  snapshot_last_key: t.Null.or(hexBuffer)
+  snapshot_last_key: t.Null.or(hexBuffer),
+  /**
+   * Source-specific metadata. Null for legacy records.
+   */
+  source_metadata: t.Null.or(jsonb_raw<JsonValue>())
 });
 
 export type SourceTable = t.Encoded<typeof SourceTable>;

package/src/types/models/WriteCheckpoint.ts

@@ -1,10 +1,12 @@
+import { framework } from '@powersync/service-core';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 
 export const WriteCheckpoint = t.object({
   user_id: t.string,
   lsns: jsonb(t.record(t.string)),
-  write_checkpoint: bigint
+  write_checkpoint: bigint,
+  checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 
 export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
@@ -13,7 +15,8 @@ export type WriteCheckpointDecoded = t.Decoded<typeof WriteCheckpoint>;
 export const CustomWriteCheckpoint = t.object({
   user_id: t.string,
   write_checkpoint: bigint,
-  sync_rules_id: bigint
+  sync_rules_id: bigint,
+  checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 
 export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;

package/src/utils/checkpoints.ts

@@ -0,0 +1,31 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { storage } from '@powersync/service-core';
+import { models } from '../types/types.js';
+
+/**
+ * Gets the latest active checkpoint document.
+ * This is mainly exported for mocking in tests.
+ */
+export async function getActiveCheckpointDocument({
+  db
+}: {
+  db: lib_postgres.DatabaseClient;
+}): Promise<models.ActiveCheckpointDecoded | null> {
+  return db.sql`
+    SELECT
+      id,
+      last_checkpoint,
+      last_checkpoint_lsn
+    FROM
+      sync_rules
+    WHERE
+      state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+      OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+    ORDER BY
+      id DESC
+    LIMIT
+      1
+  `
+    .decoded(models.ActiveCheckpoint)
+    .first();
+}