@powersync/service-module-postgres-storage 0.16.3 → 0.17.0

package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js

@@ -21,42 +21,152 @@ export class PostgresWriteCheckpointAPI {
         if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
             throw new framework.errors.ValidationError(`Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`);
         }
-        const uniqueCheckpoints = [...new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint])).values()];
+        const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints);
         if (uniqueCheckpoints.length == 0) {
-            return new Map();
+            return { writeCheckpoints: new Map(), shouldAdvance: false };
         }
-        const mappedCheckpoints = uniqueCheckpoints.map((checkpoint) => ({
-            user_id: checkpoint.user_id,
-            lsns: checkpoint.heads
-        }));
-        const rows = await this.db.sql `
-      WITH
-        json_data AS (
+        const writeCheckpoints = new Map();
+        const generatedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id == null);
+        const suppliedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id != null);
+        if (generatedCheckpoints.length > 0) {
+            const mappedCheckpoints = generatedCheckpoints.map((checkpoint) => ({
+                user_id: checkpoint.user_id,
+                lsns: checkpoint.heads
+            }));
+            const generatedRows = await this.db.sql `
+        WITH
+          json_data AS (
+            SELECT
+            CHECKPOINT ->> 'user_id' AS user_id,
+            CHECKPOINT -> 'lsns' AS lsns
+            FROM
+              jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
+            CHECKPOINT
+          )
+        INSERT INTO
+          write_checkpoints (
+            user_id,
+            lsns,
+            write_checkpoint,
+            checkpoint_requested_at
+          )
+        SELECT
+          user_id,
+          lsns,
+          1,
+          NULL
+        FROM
+          json_data
+        ON CONFLICT (user_id) DO UPDATE
+        SET
+          write_checkpoint = write_checkpoints.write_checkpoint + 1,
+          lsns = EXCLUDED.lsns,
+          checkpoint_requested_at = NULL
+        RETURNING
+          *;
+      `
+                .decoded(models.WriteCheckpoint)
+                .rows();
+            for (const row of generatedRows) {
+                writeCheckpoints.set(row.user_id, row.write_checkpoint);
+            }
+        }
+        if (suppliedCheckpoints.length > 0) {
+            // Supplied request ids are monotonic. Greater values update the checkpoint
+            // id and heads, while equal retries only refresh the retention timestamp.
+            // Stale requests return the stored id without changing the row.
+            const mappedCheckpoints = suppliedCheckpoints.map((checkpoint) => ({
+                user_id: checkpoint.user_id,
+                lsns: checkpoint.heads,
+                checkpoint_request_id: String(checkpoint.checkpoint_request_id)
+            }));
+            const suppliedRows = await this.db.sql `
+        WITH
+          json_data AS (
+            SELECT
+            CHECKPOINT ->> 'user_id' AS user_id,
+            CHECKPOINT -> 'lsns' AS lsns,
+            (
+              CHECKPOINT ->> 'checkpoint_request_id'
+            )::int8 AS checkpoint_request_id
+            FROM
+              jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
+            CHECKPOINT
+          )
+        INSERT INTO
+          write_checkpoints (
+            user_id,
+            lsns,
+            write_checkpoint,
+            checkpoint_requested_at
+          )
+        SELECT
+          user_id,
+          lsns,
+          checkpoint_request_id,
+          NOW()
+        FROM
+          json_data
+        ON CONFLICT (user_id) DO UPDATE
+        SET
+          write_checkpoint = CASE
+            WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.write_checkpoint
+            ELSE write_checkpoints.write_checkpoint
+          END,
+          lsns = CASE
+            WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.lsns
+            ELSE write_checkpoints.lsns
+          END,
+          checkpoint_requested_at = NOW()
+        WHERE
+          EXCLUDED.write_checkpoint >= write_checkpoints.write_checkpoint
+        RETURNING
+          *;
+      `
+                .decoded(models.WriteCheckpoint)
+                .rows();
+            for (const row of suppliedRows) {
+                writeCheckpoints.set(row.user_id, row.write_checkpoint);
+            }
+            // RETURNING only excludes stale requests rejected by the monotonic
+            // conflict condition. Those requests still need the stored id.
+            const returnedSuppliedUserIds = new Set(suppliedRows.map((row) => row.user_id));
+            const unchangedUserIds = suppliedCheckpoints
+                .map((checkpoint) => checkpoint.user_id)
+                .filter((userId) => !returnedSuppliedUserIds.has(userId));
+            if (unchangedUserIds.length > 0) {
+                const mappedUserIds = unchangedUserIds.map((user_id) => ({ user_id }));
+                const unchangedRows = await this.db.sql `
+          WITH
+            json_data AS (
+              SELECT
+              CHECKPOINT ->> 'user_id' AS user_id
+              FROM
+                jsonb_array_elements(${{ type: 'jsonb', value: mappedUserIds }}) AS
+              CHECKPOINT
+            )
           SELECT
-          CHECKPOINT ->> 'user_id' AS user_id,
-          CHECKPOINT -> 'lsns' AS lsns
+            write_checkpoints.*
           FROM
-            jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
-          CHECKPOINT
-        )
-      INSERT INTO
-        write_checkpoints (user_id, lsns, write_checkpoint)
-      SELECT
-        user_id,
-        lsns,
-        1
-      FROM
-        json_data
-      ON CONFLICT (user_id) DO UPDATE
-      SET
-        write_checkpoint = write_checkpoints.write_checkpoint + 1,
-        lsns = EXCLUDED.lsns
-      RETURNING
-        *;
-    `
-            .decoded(models.WriteCheckpoint)
-            .rows();
-        return new Map(rows.map((row) => [row.user_id, row.write_checkpoint]));
+            write_checkpoints
+            JOIN json_data ON write_checkpoints.user_id = json_data.user_id;
+        `
+                    .decoded(models.WriteCheckpoint)
+                    .rows();
+                for (const row of unchangedRows) {
+                    writeCheckpoints.set(row.user_id, row.write_checkpoint);
+                }
+            }
+        }
+        // Postgres storage does not track a per-row processed indicator: a write
+        // checkpoint is considered processed at read time by comparing its lsns
+        // against the replicated head (see lastManagedWriteCheckpoint). We therefore
+        // force the source marker whenever any checkpoint was matched, which also
+        // covers stale or duplicate requests whose stored checkpoint may still be
+        // pending. Forcing a marker for an already-processed checkpoint is wasteful
+        // but harmless.
+        const shouldAdvance = generatedCheckpoints.length > 0 || suppliedCheckpoints.length > 0;
+        return { writeCheckpoints, shouldAdvance };
     }
     async lastWriteCheckpoint(filters) {
         switch (this.writeCheckpointMode) {
@@ -120,7 +230,8 @@ export async function batchCreateCustomWriteCheckpoints(db, checkpoints) {
             // Cannot encode bigint directly using JSON.stringify.
             // The ::int8 in the query below will take care of casting back to a number
             checkpoint: String(cp.checkpoint),
-            sync_rules_id: cp.sync_rules_id
+            sync_rules_id: cp.sync_rules_id,
+            checkpoint_requested_at: cp.checkpoint_requested_at?.toISOString() ?? null
         };
     });
     await db.sql `
@@ -131,7 +242,12 @@ export async function batchCreateCustomWriteCheckpoints(db, checkpoints) {
         CHECKPOINT
       )
     INSERT INTO
-      custom_write_checkpoints (user_id, write_checkpoint, sync_rules_id)
+      custom_write_checkpoints (
+        user_id,
+        write_checkpoint,
+        sync_rules_id,
+        checkpoint_requested_at
+      )
     SELECT
     CHECKPOINT ->> 'user_id'::varchar,
     (
@@ -139,12 +255,16 @@ export async function batchCreateCustomWriteCheckpoints(db, checkpoints) {
     )::int8,
     (
       CHECKPOINT ->> 'sync_rules_id'
-    )::int4
+    )::int4,
+    (
+      CHECKPOINT ->> 'checkpoint_requested_at'
+    )::timestamptz
     FROM
       json_data
     ON CONFLICT (user_id, sync_rules_id) DO UPDATE
     SET
-      write_checkpoint = EXCLUDED.write_checkpoint;
+      write_checkpoint = EXCLUDED.write_checkpoint,
+      checkpoint_requested_at = EXCLUDED.checkpoint_requested_at;
   `.execute();
 }
 //# sourceMappingURL=PostgresWriteCheckpointAPI.js.map

package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map

@@ -1 +1 @@
-{"version":3,"file":"PostgresWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/checkpoints/PostgresWriteCheckpointAPI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAO9C,MAAM,OAAO,0BAA0B;IAC5B,EAAE,CAA8B;IACjC,KAAK,CAA8B;IAE3C,YAAY,OAAqC;QAC/C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,iCAAiC,CACrC,WAAmD,EACnD,KAAmB;QAEnB,OAAO,iCAAiC,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,6BAA6B,CACjC,WAAoD;QAEpD,IAAI,IAAI,CAAC,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qGAAqG,IAAI,CAAC,mBAAmB,GAAG,CACjI,CAAC;QACJ,CAAC;QAED,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACnH,IAAI,iBAAiB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC/D,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,IAAI,EAAE,UAAU,CAAC,KAAK;SACvB,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;mCAOC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE;;;;;;;;;;;;;;;;;KAiBzE;aACE,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;aAC/B,IAAI,EAAE,CAAC;QAEV,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAA2C;QACnE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,IAAI,KAAK,IAAI,eAAe,IAAI,OAAO,EAAE,CAAC;oBACxC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,yEAAyE,CAC1E,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAA+C,CAAC,CAAC;YACzF,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qEAAqE,CACtE,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAgD,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IAES,KAAK,CAAC,yBAAyB,CAAC,OAA6C;QACrF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;oBAMb,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;8BACzB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE;KAC/D;aACE,OAAO,CAAC,MAAM,CAAC,qBAAqB,CAAC;aACrC,KAAK,EAAE,CAAC;QACX,OAAO,GAAG,EAAE,gBAAgB,IAAI,IAAI,CAAC;IACvC,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAA8C;QACvF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACnC,4EAA4E;QAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,oDAAoD;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;oBAMb,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;8BACzB,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;KACxD;aACE,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;aAC/B,KAAK,EAAE,CAAC;QACX,OAAO,GAAG,EAAE,gBAAgB,IAAI,IAAI,CAAC;IACvC,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,EAA+B,EAC/B,WAAmD;IAEnD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;IAED,iDAAiD;IACjD,MAAM,iBAAiB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/C,OAAO;YACL,OAAO,EAAE,EAAE,CAAC,OAAO;YACnB,sDAAsD;YACtD,2EAA2E;YAC3E,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC;YACjC,aAAa,EAAE,EAAE,CAAC,aAAa;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,EAAE,CAAC,GAAG,CAAA;;;;iCAImB,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE;;;;;;;;;;;;;;;;;;GAkBzE,CAAC,OAAO,EAAE,CAAC;AACd,CAAC"}
+{"version":3,"file":"PostgresWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/checkpoints/PostgresWriteCheckpointAPI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAO9C,MAAM,OAAO,0BAA0B;IAC5B,EAAE,CAA8B;IACjC,KAAK,CAA8B;IAE3C,YAAY,OAAqC;QAC/C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,iCAAiC,CACrC,WAAmD,EACnD,KAAmB;QAEnB,OAAO,iCAAiC,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,6BAA6B,CACjC,WAAoD;QAEpD,IAAI,IAAI,CAAC,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qGAAqG,IAAI,CAAC,mBAAmB,GAAG,CACjI,CAAC;QACJ,CAAC;QAED,MAAM,iBAAiB,GAAG,OAAO,CAAC,6BAA6B,CAAC,WAAW,CAAC,CAAC;QAC7E,IAAI,iBAAiB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,gBAAgB,EAAE,IAAI,GAAG,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACnD,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,qBAAqB,IAAI,IAAI,CAAC,CAAC;QAChH,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,qBAAqB,IAAI,IAAI,CAAC,CAAC;QAE/G,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBAClE,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,IAAI,EAAE,UAAU,CAAC,KAAK;aACvB,CAAC,CAAC,CAAC;YAEJ,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;qCAOR,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE;;;;;;;;;;;;;;;;;;;;;;;;OAwBzE;iBACE,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;iBAC/B,IAAI,EAAE,CAAC;YAEV,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,2EAA2E;YAC3E,0EAA0E;YAC1E,gEAAgE;YAChE,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBACjE,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,IAAI,EAAE,UAAU,CAAC,KAAK;gBACtB,qBAAqB,EAAE,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC;aAChE,CAAC,CAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;qCAUP,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCzE;iBACE,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;iBAC/B,IAAI,EAAE,CAAC;YAEV,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;gBAC/B,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC1D,CAAC;YAED,mEAAmE;YACnE,+DAA+D;YAC/D,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YAChF,MAAM,gBAAgB,GAAG,mBAAmB;iBACzC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;iBACvC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YAE5D,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;uCAMR,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE;;;;;;;;SAQrE;qBACE,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;qBAC/B,IAAI,EAAE,CAAC;gBAEV,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;oBAChC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;QACH,CAAC;QAED,yEAAyE;QACzE,wEAAwE;QACxE,6EAA6E;QAC7E,0EAA0E;QAC1E,0EAA0E;QAC1E,4EAA4E;QAC5E,gBAAgB;QAChB,MAAM,aAAa,GAAG,oBAAoB,CAAC,MAAM,GAAG,CAAC,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC;QACxF,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAA2C;QACnE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,IAAI,KAAK,IAAI,eAAe,IAAI,OAAO,EAAE,CAAC;oBACxC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,yEAAyE,CAC1E,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAA+C,CAAC,CAAC;YACzF,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qEAAqE,CACtE,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAgD,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IAES,KAAK,CAAC,yBAAyB,CAAC,OAA6C;QACrF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;oBAMb,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;8BACzB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE;KAC/D;aACE,OAAO,CAAC,MAAM,CAAC,qBAAqB,CAAC;aACrC,KAAK,EAAE,CAAC;QACX,OAAO,GAAG,EAAE,gBAAgB,IAAI,IAAI,CAAC;IACvC,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAA8C;QACvF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACnC,4EAA4E;QAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,oDAAoD;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;oBAMb,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;8BACzB,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;KACxD;aACE,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;aAC/B,KAAK,EAAE,CAAC;QACX,OAAO,GAAG,EAAE,gBAAgB,IAAI,IAAI,CAAC;IACvC,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,EAA+B,EAC/B,WAAmD;IAEnD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;IAED,iDAAiD;IACjD,MAAM,iBAAiB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/C,OAAO;YACL,OAAO,EAAE,EAAE,CAAC,OAAO;YACnB,sDAAsD;YACtD,2EAA2E;YAC3E,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC;YACjC,aAAa,EAAE,EAAE,CAAC,aAAa;YAC/B,uBAAuB,EAAE,EAAE,CAAC,uBAAuB,EAAE,WAAW,EAAE,IAAI,IAAI;SAC3E,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,EAAE,CAAC,GAAG,CAAA;;;;iCAImB,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BzE,CAAC,OAAO,EAAE,CAAC;AACd,CAAC"}

package/dist/storage/current-data-store.d.ts

@@ -77,9 +77,15 @@ export declare class PostgresCurrentDataStore {
         groupId: number;
         lastCheckpoint: bigint;
     }): Promise<void>;
-    deleteGroupRows(db: Queryable, options: {
+    /**
+     * Delete up to `limit` rows for the group, returning the number of candidate
+     * rows found by the scan (see PostgresSyncRulesStorage#deleteGroupBatch for
+     * why the scan is counted rather than the delete).
+     */
+    deleteGroupRowsBatch(db: Queryable, options: {
         groupId: number;
-    }): Promise<void>;
+        limit: number;
+    }): Promise<bigint>;
     private wherePendingDelete;
 }
 export {};

package/dist/storage/current-data-store.js

@@ -244,20 +244,75 @@ export class PostgresCurrentDataStore {
         AND pending_delete <= ${{ type: 'int8', value: options.lastCheckpoint }}
     `.execute();
     }
-    async deleteGroupRows(db, options) {
+    /**
+     * Delete up to `limit` rows for the group, returning the number of candidate
+     * rows found by the scan (see PostgresSyncRulesStorage#deleteGroupBatch for
+     * why the scan is counted rather than the delete).
+     */
+    async deleteGroupRowsBatch(db, options) {
         if (this.softDeleteEnabled) {
-            await db.sql `
-        DELETE FROM v3_current_data
-        WHERE
-          group_id = ${{ type: 'int4', value: options.groupId }}
-      `.execute();
+            const result = await db.sql `
+        WITH
+          batch AS (
+            SELECT
+              ctid
+            FROM
+              v3_current_data
+            WHERE
+              group_id = ${{ type: 'int4', value: options.groupId }}
+            LIMIT
+              ${{ type: 'int4', value: options.limit }}
+          ),
+          deleted AS (
+            DELETE FROM v3_current_data
+            WHERE
+              ctid IN (
+                SELECT
+                  ctid
+                FROM
+                  batch
+              )
+            RETURNING
+              1
+          )
+        SELECT
+          COUNT(*) AS count
+        FROM
+          batch
+      `.first();
+            return result?.count ?? 0n;
         }
         else {
-            await db.sql `
-        DELETE FROM current_data
-        WHERE
-          group_id = ${{ type: 'int4', value: options.groupId }}
-      `.execute();
+            const result = await db.sql `
+        WITH
+          batch AS (
+            SELECT
+              ctid
+            FROM
+              current_data
+            WHERE
+              group_id = ${{ type: 'int4', value: options.groupId }}
+            LIMIT
+              ${{ type: 'int4', value: options.limit }}
+          ),
+          deleted AS (
+            DELETE FROM current_data
+            WHERE
+              ctid IN (
+                SELECT
+                  ctid
+                FROM
+                  batch
+              )
+            RETURNING
+              1
+          )
+        SELECT
+          COUNT(*) AS count
+        FROM
+          batch
+      `.first();
+            return result?.count ?? 0n;
         }
     }
     wherePendingDelete(options) {

package/dist/storage/current-data-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"current-data-store.js","sourceRoot":"","sources":["../../src/storage/current-data-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,MAAM,MAAM,gCAAgC,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAI5C,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;AAClG,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;AAQlF,MAAM,CAAC,MAAM,qBAAqB,GAAG,cAAc,CAAC;AACpD,MAAM,CAAC,MAAM,qBAAqB,GAAG,iBAAiB,CAAC;AAEvD,MAAM,OAAO,wBAAwB;IAC1B,KAAK,CAAS;IACd,iBAAiB,CAAU;IAEpC,YAAY,aAA2C;QACrD,IAAI,CAAC,iBAAiB,GAAG,aAAa,CAAC,qBAAqB,CAAC;QAC7D,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC;IACnG,CAAC;IAED,kBAAkB,CAChB,EAAa,EACb,OAIC;QAED,OAAO,EAAE,CAAC,UAAU,CAA6C;YAC/D,SAAS,EAAE;;;;;;YAML,IAAI,CAAC,KAAK;;;;YAIV,IAAI,CAAC,kBAAkB,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;;;;OAIpD;YACD,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;gBACxC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,EAAE;gBACjD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;aACvC;SACF,CAAC,CAAC;IACL,CAAC;IAED,iBAAiB,CAAC,GAA+C;QAC/D,OAAO,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IAED,cAAc,CACZ,EAAa,EACb,OAGC;QAED,OAAO,EAAE,CAAC,UAAU,CAIjB;YACD,SAAS,EAAE;;;;;;;;;;;;;;YAcL,IAAI,CAAC,KAAK;;;;;;OAMf;YACD,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;gBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;aACzC;SACF,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CACd,EAAa,EACb,OAIC;QAED,MAAM,aAAa,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,KAAK,CAAC;QACxF,OAAO,EAAE,CAAC,UAAU,CAAM;YACxB,SAAS,EAAE;;YAEL,aAAa;;YAEb,IAAI,CAAC,KAAK;;;;;;;;;;;;OAYf;YACD,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;gBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;aACzC;SACF,CAAC,CAAC;IACL,CAAC;IAED,eAAe,CAAC,GAAQ,EAAE,gBAAyB;QACjD,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtG,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAa,EAAE,OAA+B;QAC/D,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;8BA4BY,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE;;;;;;;;;;;;;;;OAevD,CAAC,OAAO,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;;;;;;;;;;;;4BAuBY,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE;;;;;;;;;;;;;KAavD,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,EAAa,EACb,OAGC;QAED,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;mCAOiB;gBACzB,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,OAAO,CAAC,OAAO;aACvB;;;;uCAI8B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;;OAGxE,CAAC,OAAO,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;iCAOiB;YACzB,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,OAAO,CAAC,OAAO;SACvB;;;;kCAI2B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;;KAGrE,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,EAAa,EAAE,OAAoD;QAC7F,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;QACD,MAAM,EAAE,CAAC,GAAG,CAAA;;;qBAGK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;gCAE7B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE;KAC1E,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAa,EAAE,OAA4B;QAC/D,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,GAAG,CAAA;;;uBAGK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;OACxD,CAAC,OAAO,EAAE,CAAC;QACd,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,CAAC,GAAG,CAAA;;;uBAGK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;OACxD,CAAC,OAAO,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAEO,kBAAkB,CAAC,OAAkC;QAC3D,IAAI,IAAI,CAAC,iBAAiB,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACnD,OAAO,4BAA4B,CAAC;QACtC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}
+{"version":3,"file":"current-data-store.js","sourceRoot":"","sources":["../../src/storage/current-data-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,MAAM,MAAM,gCAAgC,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAI5C,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;AAClG,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;AAQlF,MAAM,CAAC,MAAM,qBAAqB,GAAG,cAAc,CAAC;AACpD,MAAM,CAAC,MAAM,qBAAqB,GAAG,iBAAiB,CAAC;AAEvD,MAAM,OAAO,wBAAwB;IAC1B,KAAK,CAAS;IACd,iBAAiB,CAAU;IAEpC,YAAY,aAA2C;QACrD,IAAI,CAAC,iBAAiB,GAAG,aAAa,CAAC,qBAAqB,CAAC;QAC7D,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC;IACnG,CAAC;IAED,kBAAkB,CAChB,EAAa,EACb,OAIC;QAED,OAAO,EAAE,CAAC,UAAU,CAA6C;YAC/D,SAAS,EAAE;;;;;;YAML,IAAI,CAAC,KAAK;;;;YAIV,IAAI,CAAC,kBAAkB,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;;;;OAIpD;YACD,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;gBACxC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,EAAE;gBACjD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;aACvC;SACF,CAAC,CAAC;IACL,CAAC;IAED,iBAAiB,CAAC,GAA+C;QAC/D,OAAO,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IAED,cAAc,CACZ,EAAa,EACb,OAGC;QAED,OAAO,EAAE,CAAC,UAAU,CAIjB;YACD,SAAS,EAAE;;;;;;;;;;;;;;YAcL,IAAI,CAAC,KAAK;;;;;;OAMf;YACD,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;gBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;aACzC;SACF,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CACd,EAAa,EACb,OAIC;QAED,MAAM,aAAa,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,KAAK,CAAC;QACxF,OAAO,EAAE,CAAC,UAAU,CAAM;YACxB,SAAS,EAAE;;YAEL,aAAa;;YAEb,IAAI,CAAC,KAAK;;;;;;;;;;;;OAYf;YACD,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;gBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;aACzC;SACF,CAAC,CAAC;IACL,CAAC;IAED,eAAe,CAAC,GAAQ,EAAE,gBAAyB;QACjD,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtG,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAa,EAAE,OAA+B;QAC/D,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;8BA4BY,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE;;;;;;;;;;;;;;;OAevD,CAAC,OAAO,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;;;;;;;;;;;;4BAuBY,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE;;;;;;;;;;;;;KAavD,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,EAAa,EACb,OAGC;QAED,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;mCAOiB;gBACzB,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,OAAO,CAAC,OAAO;aACvB;;;;uCAI8B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;;OAGxE,CAAC,OAAO,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;iCAOiB;YACzB,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,OAAO,CAAC,OAAO;SACvB;;;;kCAI2B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;;KAGrE,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,EAAa,EAAE,OAAoD;QAC7F,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;QACD,MAAM,EAAE,CAAC,GAAG,CAAA;;;qBAGK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;gCAE7B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE;KAC1E,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,oBAAoB,CAAC,EAAa,EAAE,OAA2C;QACnF,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;2BAQN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;gBAEnD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;;;;;;;;;;;;;;;;;;OAkB/C,CAAC,KAAK,EAAqB,CAAC;YAC7B,OAAO,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;2BAQN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE;;gBAEnD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;;;;;;;;;;;;;;;;;;OAkB/C,CAAC,KAAK,EAAqB,CAAC;YAC7B,OAAO,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,kBAAkB,CAAC,OAAkC;QAC3D,IAAI,IAAI,CAAC,iBAAiB,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACnD,OAAO,4BAA4B,CAAC;QACtC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}

package/dist/types/models/WriteCheckpoint.d.ts

@@ -3,6 +3,7 @@ export declare const WriteCheckpoint: t.ObjectCodec<{
     user_id: t.IdentityCodec<t.CodecType.String>;
     lsns: t.Codec<Record<string, string>, string, string, t.CodecProps>;
     write_checkpoint: t.Codec<bigint, string | number, string, t.CodecProps>;
+    checkpoint_requested_at: t.Union<t.Codec<null, null, string, t.CodecProps>, t.Codec<Date, string | import("@powersync/service-sync-rules").DateTimeValue, string, t.CodecProps>>;
 }>;
 export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
 export type WriteCheckpointDecoded = t.Decoded<typeof WriteCheckpoint>;
@@ -10,6 +11,7 @@ export declare const CustomWriteCheckpoint: t.ObjectCodec<{
     user_id: t.IdentityCodec<t.CodecType.String>;
     write_checkpoint: t.Codec<bigint, string | number, string, t.CodecProps>;
     sync_rules_id: t.Codec<bigint, string | number, string, t.CodecProps>;
+    checkpoint_requested_at: t.Union<t.Codec<null, null, string, t.CodecProps>, t.Codec<Date, string | import("@powersync/service-sync-rules").DateTimeValue, string, t.CodecProps>>;
 }>;
 export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;
 export type CustomWriteCheckpointDecoded = t.Decoded<typeof CustomWriteCheckpoint>;

package/dist/types/models/WriteCheckpoint.js

@@ -1,13 +1,16 @@
+import { framework } from '@powersync/service-core';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 export const WriteCheckpoint = t.object({
     user_id: t.string,
     lsns: jsonb(t.record(t.string)),
-    write_checkpoint: bigint
+    write_checkpoint: bigint,
+    checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 export const CustomWriteCheckpoint = t.object({
     user_id: t.string,
     write_checkpoint: bigint,
-    sync_rules_id: bigint
+    sync_rules_id: bigint,
+    checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 //# sourceMappingURL=WriteCheckpoint.js.map

package/dist/types/models/WriteCheckpoint.js.map

@@ -1 +1 @@
-{"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;CACzB,CAAC,CAAC;AAKH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,gBAAgB,EAAE,MAAM;IACxB,aAAa,EAAE,MAAM;CACtB,CAAC,CAAC"}
+{"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;IACxB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC;AAKH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,gBAAgB,EAAE,MAAM;IACxB,aAAa,EAAE,MAAM;IACrB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC"}

package/dist/utils/checkpoints.d.ts

@@ -0,0 +1,9 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { models } from '../types/types.js';
+/**
+ * Gets the latest active checkpoint document.
+ * This is mainly exported for mocking in tests.
+ */
+export declare function getActiveCheckpointDocument({ db }: {
+    db: lib_postgres.DatabaseClient;
+}): Promise<models.ActiveCheckpointDecoded | null>;

package/dist/utils/checkpoints.js

@@ -0,0 +1,26 @@
+import { storage } from '@powersync/service-core';
+import { models } from '../types/types.js';
+/**
+ * Gets the latest active checkpoint document.
+ * This is mainly exported for mocking in tests.
+ */
+export async function getActiveCheckpointDocument({ db }) {
+    return db.sql `
+    SELECT
+      id,
+      last_checkpoint,
+      last_checkpoint_lsn
+    FROM
+      sync_rules
+    WHERE
+      state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+      OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+    ORDER BY
+      id DESC
+    LIMIT
+      1
+  `
+        .decoded(models.ActiveCheckpoint)
+        .first();
+}
+//# sourceMappingURL=checkpoints.js.map

package/dist/utils/checkpoints.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoints.js","sourceRoot":"","sources":["../../src/utils/checkpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,EAChD,EAAE,EAGH;IACC,OAAO,EAAE,CAAC,GAAG,CAAA;;;;;;;;gBAQC,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;mBACrD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;;;;;GAKzE;SACE,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;SAChC,KAAK,EAAE,CAAC;AACb,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@powersync/service-module-postgres-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "version": "0.16.3",
+  "version": "0.17.0",
   "license": "FSL-1.1-ALv2",
   "type": "module",
   "publishConfig": {
@@ -16,17 +16,17 @@
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^14.0.0",
-    "@powersync/lib-service-postgres": "0.5.2",
-    "@powersync/lib-services-framework": "0.9.8",
-    "@powersync/service-core": "1.23.3",
-    "@powersync/service-types": "0.16.1",
-    "@powersync/service-jpgwire": "0.21.21",
+    "@powersync/lib-service-postgres": "0.5.3",
+    "@powersync/lib-services-framework": "0.10.0",
+    "@powersync/service-core": "1.24.0",
+    "@powersync/service-types": "0.17.0",
     "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-sync-rules": "0.39.0"
+    "@powersync/service-jpgwire": "0.21.22",
+    "@powersync/service-sync-rules": "0.40.0"
   },
   "devDependencies": {
     "typescript": "~6.0.3",
-    "@powersync/service-core-tests": "0.17.2"
+    "@powersync/service-core-tests": "0.18.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts

@@ -0,0 +1,51 @@
+import { migrations } from '@powersync/service-core';
+
+import { openMigrationDB } from '../migration-utils.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  await client.transaction(async (db) => {
+    await db.sql`
+      ALTER TABLE write_checkpoints
+      ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
+    `.execute();
+    await db.sql`
+      ALTER TABLE custom_write_checkpoints
+      ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
+    `.execute();
+    await db.sql`
+      CREATE INDEX write_checkpoints_checkpoint_requested_at ON write_checkpoints (checkpoint_requested_at)
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+    `.execute();
+    await db.sql`
+      CREATE INDEX custom_write_checkpoints_checkpoint_requested_at ON custom_write_checkpoints (checkpoint_requested_at)
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+    `.execute();
+  });
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  await client.transaction(async (db) => {
+    // Also drops the write_checkpoints_checkpoint_requested_at index.
+    await db.sql`
+      ALTER TABLE write_checkpoints
+      DROP COLUMN checkpoint_requested_at
+    `.execute();
+    // Also drops the custom_write_checkpoints_checkpoint_requested_at index.
+    await db.sql`
+      ALTER TABLE custom_write_checkpoints
+      DROP COLUMN checkpoint_requested_at
+    `.execute();
+  });
+};

package/src/storage/PostgresBucketStorageFactory.ts

@@ -8,7 +8,6 @@ import { models, NormalizedPostgresStorageConfig } from '../types/types.js';
 
 import { getStorageApplicationName } from '../utils/application-name.js';
 import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js';
-import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js';
 import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js';
 import { PostgresPersistedReplicationStream } from './sync-rules/PostgresPersistedSyncConfigContent.js';
 
@@ -220,8 +219,6 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
         .decoded(models.SyncRules)
         .first();
 
-      await notifySyncRulesUpdate(this.db, newSyncRulesRow!);
-
       return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
     });
   }

package/src/storage/PostgresCompactor.ts

@@ -53,6 +53,7 @@ export class PostgresCompactor {
   private clearBatchLimit: number;
   private maxOpId: InternalOpId;
   private buckets: string[] | undefined;
+  private deleteCheckpointRequestsBefore: Date | undefined;
 
   constructor(
     private db: lib_postgres.DatabaseClient,
@@ -65,6 +66,7 @@ export class PostgresCompactor {
     this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
     this.maxOpId = options.maxOpId ?? 0n;
     this.buckets = options.compactBuckets;
+    this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
   }
 
   /**
@@ -73,6 +75,8 @@ export class PostgresCompactor {
    * See /docs/storage/compacting-operations.md for details.
    */
   async compact() {
+    await this.deleteOldCheckpointRequests();
+
     if (this.buckets) {
       for (let bucket of this.buckets) {
         await this.compactSingleBucket(bucket);
@@ -82,6 +86,26 @@ export class PostgresCompactor {
     }
   }
 
+  private async deleteOldCheckpointRequests() {
+    if (this.deleteCheckpointRequestsBefore == null) {
+      return;
+    }
+
+    await this.db.sql`
+      DELETE FROM write_checkpoints
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+        AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
+    `.execute();
+
+    await this.db.sql`
+      DELETE FROM custom_write_checkpoints
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+        AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
+    `.execute();
+  }
+
   private async compactAllBuckets() {
     const DISCOVERY_BATCH_SIZE = 200;
     let lastBucket = '';