@powersync/service-module-postgres-storage 0.1.2 → 0.2.1

package/src/storage/batch/PostgresBucketBatch.ts

@@ -89,7 +89,7 @@ export class PostgresBucketBatch
 
   async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
     // TODO maybe share with abstract class
-    const { after, afterReplicaId, before, beforeReplicaId, sourceTable, tag } = record;
+    const { after, before, sourceTable, tag } = record;
     for (const event of this.getTableEvents(sourceTable)) {
       this.iterateListeners((cb) =>
         cb.replicationEvent?.({
@@ -245,7 +245,10 @@ export class PostgresBucketBatch
 
   private async flushInner(): Promise<storage.FlushedResult | null> {
     const batch = this.batch;
-    if (batch == null) {
+    // Don't flush empty batches
+    // This helps prevent feedback loops when using the same database for
+    // the source data and sync bucket storage
+    if (batch == null || batch.length == 0) {
       return null;
     }
 
@@ -275,7 +278,9 @@ export class PostgresBucketBatch
     return { flushed_op: String(lastOp) };
   }
 
-  async commit(lsn: string): Promise<boolean> {
+  async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<boolean> {
+    const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
+
     await this.flush();
 
     if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
@@ -309,6 +314,12 @@ export class PostgresBucketBatch
 
       return false;
     }
+
+    // Don't create a checkpoint if there were no changes
+    if (!createEmptyCheckpoints && this.persisted_op == null) {
+      return false;
+    }
+
     const now = new Date().toISOString();
     const update: Partial<models.SyncRules> = {
       last_checkpoint_lsn: lsn,
@@ -488,7 +499,7 @@ export class PostgresBucketBatch
               jsonb_array_elements(${{ type: 'jsonb', value: sizeLookups }}::jsonb) AS FILTER
           )
         SELECT
-          pg_column_size(c.data) AS data_size,
+          octet_length(c.data) AS data_size,
           c.source_table,
           c.source_key
         FROM
@@ -529,23 +540,20 @@ export class PostgresBucketBatch
       const current_data_lookup = new Map<string, CurrentDataDecoded>();
       for await (const currentDataRows of db.streamRows<CurrentData>({
         statement: /* sql */ `
-          WITH
-            filter_data AS (
-              SELECT
-                decode(FILTER ->> 'source_key', 'hex') AS source_key, -- Decoding from hex to bytea
-                (FILTER ->> 'source_table') AS source_table_id
-              FROM
-                jsonb_array_elements($1::jsonb) AS FILTER
-            )
           SELECT
-            --- With skipExistingRows, we only need to know whether or not the row exists.
             ${this.options.skip_existing_rows ? `c.source_table, c.source_key` : 'c.*'}
           FROM
             current_data c
-            JOIN filter_data f ON c.source_table = f.source_table_id
+            JOIN (
+              SELECT
+                decode(FILTER ->> 'source_key', 'hex') AS source_key,
+                FILTER ->> 'source_table' AS source_table_id
+              FROM
+                jsonb_array_elements($1::jsonb) AS FILTER
+            ) f ON c.source_table = f.source_table_id
             AND c.source_key = f.source_key
           WHERE
-            c.group_id = $2
+            c.group_id = $2;
         `,
         params: [
           {
@@ -553,7 +561,7 @@ export class PostgresBucketBatch
             value: lookups
           },
           {
-            type: 'int8',
+            type: 'int4',
             value: this.group_id
           }
         ]
@@ -610,7 +618,12 @@ export class PostgresBucketBatch
         await persistedBatch.flush(db);
       }
     }
-    return resumeBatch;
+
+    // Don't return empty batches
+    if (resumeBatch?.batch.length) {
+      return resumeBatch;
+    }
+    return null;
   }
 
   protected async saveOperation(

package/src/storage/batch/PostgresPersistedBatch.ts

@@ -256,33 +256,6 @@ export class PostgresPersistedBatch {
   protected async flushBucketData(db: lib_postgres.WrappedConnection) {
     if (this.bucketDataInserts.length > 0) {
       await db.sql`
-        WITH
-          parsed_data AS (
-            SELECT
-              group_id,
-              bucket_name,
-              source_table,
-              decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
-              table_name,
-              op,
-              row_id,
-              checksum,
-              data,
-              target_op
-            FROM
-              jsonb_to_recordset(${{ type: 'jsonb', value: this.bucketDataInserts }}::jsonb) AS t (
-                group_id integer,
-                bucket_name text,
-                source_table text,
-                source_key text, -- Input as hex string
-                table_name text,
-                op text,
-                row_id text,
-                checksum bigint,
-                data text,
-                target_op bigint
-              )
-          )
         INSERT INTO
           bucket_data (
             group_id,
@@ -303,14 +276,25 @@ export class PostgresPersistedBatch {
           nextval('op_id_sequence'),
           op,
           source_table,
-          source_key, -- Already decoded
+          decode(source_key, 'hex') AS source_key,
           table_name,
           row_id,
           checksum,
           data,
           target_op
         FROM
-          parsed_data;
+          json_to_recordset(${{ type: 'json', value: this.bucketDataInserts }}::json) AS t (
+            group_id integer,
+            bucket_name text,
+            source_table text,
+            source_key text, -- Input as hex string
+            table_name text,
+            op text,
+            row_id text,
+            checksum bigint,
+            data text,
+            target_op bigint
+          );
       `.execute();
     }
   }
@@ -318,23 +302,6 @@ export class PostgresPersistedBatch {
   protected async flushParameterData(db: lib_postgres.WrappedConnection) {
     if (this.parameterDataInserts.length > 0) {
       await db.sql`
-        WITH
-          parsed_data AS (
-            SELECT
-              group_id,
-              source_table,
-              decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
-              decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
-              bucket_parameters
-            FROM
-              jsonb_to_recordset(${{ type: 'jsonb', value: this.parameterDataInserts }}::jsonb) AS t (
-                group_id integer,
-                source_table text,
-                source_key text, -- Input as hex string
-                lookup text, -- Input as hex string
-                bucket_parameters text -- Input as stringified JSON
-              )
-          )
         INSERT INTO
           bucket_parameters (
             group_id,
@@ -346,11 +313,17 @@ export class PostgresPersistedBatch {
         SELECT
           group_id,
           source_table,
-          source_key, -- Already decoded
-          lookup, -- Already decoded
+          decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
+          decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
           bucket_parameters
         FROM
-          parsed_data;
+          json_to_recordset(${{ type: 'json', value: this.parameterDataInserts }}::json) AS t (
+            group_id integer,
+            source_table text,
+            source_key text, -- Input as hex string
+            lookup text, -- Input as hex string
+            bucket_parameters text -- Input as stringified JSON
+          )
       `.execute();
     }
   }
@@ -358,33 +331,6 @@ export class PostgresPersistedBatch {
   protected async flushCurrentData(db: lib_postgres.WrappedConnection) {
     if (this.currentDataInserts.size > 0) {
       await db.sql`
-        WITH
-          parsed_data AS (
-            SELECT
-              group_id,
-              source_table,
-              decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
-              buckets::jsonb AS buckets,
-              decode(data, 'hex') AS data, -- Decode hex to bytea
-              ARRAY(
-                SELECT
-                  decode((value ->> 0)::TEXT, 'hex')
-                FROM
-                  jsonb_array_elements(lookups::jsonb) AS value
-              ) AS lookups -- Decode array of hex strings to bytea[]
-            FROM
-              jsonb_to_recordset(${{
-          type: 'jsonb',
-          value: Array.from(this.currentDataInserts.values())
-        }}::jsonb) AS t (
-                group_id integer,
-                source_table text,
-                source_key text, -- Input as hex string
-                buckets text,
-                data text, -- Input as hex string
-                lookups text -- Input as stringified JSONB array of hex strings
-              )
-          )
         INSERT INTO
           current_data (
             group_id,
@@ -397,12 +343,24 @@ export class PostgresPersistedBatch {
         SELECT
           group_id,
           source_table,
-          source_key, -- Already decoded
-          buckets,
-          data, -- Already decoded
-          lookups -- Already decoded
+          decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
+          buckets::jsonb AS buckets,
+          decode(data, 'hex') AS data, -- Decode hex to bytea
+          array(
+            SELECT
+              decode(element, 'hex')
+            FROM
+              unnest(lookups) AS element
+          ) AS lookups
         FROM
-          parsed_data
+          json_to_recordset(${{ type: 'json', value: Array.from(this.currentDataInserts.values()) }}::json) AS t (
+            group_id integer,
+            source_table text,
+            source_key text, -- Input as hex string
+            buckets text,
+            data text, -- Input as hex string
+            lookups TEXT[] -- Input as stringified JSONB array of hex strings
+          )
         ON CONFLICT (group_id, source_table, source_key) DO UPDATE
         SET
           buckets = EXCLUDED.buckets,