@powersync/service-module-postgres-storage 0.0.0-dev-20250117095455 → 0.0.0-dev-20250214100224

package/src/storage/batch/PostgresBucketBatch.ts

@@ -1,5 +1,14 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
-import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework';
+import {
+  container,
+  DisposableObserver,
+  ErrorCode,
+  errors,
+  logger,
+  ReplicationAssertionError,
+  ServiceAssertionError,
+  ServiceError
+} from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
@@ -80,7 +89,7 @@ export class PostgresBucketBatch
 
   async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
     // TODO maybe share with abstract class
-    const { after, afterReplicaId, before, beforeReplicaId, sourceTable, tag } = record;
+    const { after, before, sourceTable, tag } = record;
     for (const event of this.getTableEvents(sourceTable)) {
       this.iterateListeners((cb) =>
         cb.replicationEvent?.({
@@ -236,7 +245,10 @@ export class PostgresBucketBatch
 
   private async flushInner(): Promise<storage.FlushedResult | null> {
     const batch = this.batch;
-    if (batch == null) {
+    // Don't flush empty batches
+    // This helps prevent feedback loops when using the same database for
+    // the source data and sync bucket storage
+    if (batch == null || batch.length == 0) {
       return null;
     }
 
@@ -258,7 +270,7 @@ export class PostgresBucketBatch
     this.batch = resumeBatch;
 
     if (lastOp == null) {
-      throw new Error('Unexpected last_op == null');
+      throw new ServiceAssertionError('Unexpected last_op == null');
     }
 
     this.persisted_op = lastOp;
@@ -266,7 +278,9 @@ export class PostgresBucketBatch
     return { flushed_op: String(lastOp) };
   }
 
-  async commit(lsn: string): Promise<boolean> {
+  async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<boolean> {
+    const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
+
     await this.flush();
 
     if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
@@ -300,6 +314,12 @@ export class PostgresBucketBatch
 
       return false;
     }
+
+    // Don't create a checkpoint if there were no changes
+    if (!createEmptyCheckpoints && this.persisted_op == null) {
+      return false;
+    }
+
     const now = new Date().toISOString();
     const update: Partial<models.SyncRules> = {
       last_checkpoint_lsn: lsn,
@@ -479,7 +499,7 @@ export class PostgresBucketBatch
               jsonb_array_elements(${{ type: 'jsonb', value: sizeLookups }}::jsonb) AS FILTER
           )
         SELECT
-          pg_column_size(c.data) AS data_size,
+          octet_length(c.data) AS data_size,
           c.source_table,
           c.source_key
         FROM
@@ -520,23 +540,20 @@ export class PostgresBucketBatch
       const current_data_lookup = new Map<string, CurrentDataDecoded>();
       for await (const currentDataRows of db.streamRows<CurrentData>({
         statement: /* sql */ `
-          WITH
-            filter_data AS (
-              SELECT
-                decode(FILTER ->> 'source_key', 'hex') AS source_key, -- Decoding from hex to bytea
-                (FILTER ->> 'source_table') AS source_table_id
-              FROM
-                jsonb_array_elements($1::jsonb) AS FILTER
-            )
           SELECT
-            --- With skipExistingRows, we only need to know whether or not the row exists.
             ${this.options.skip_existing_rows ? `c.source_table, c.source_key` : 'c.*'}
           FROM
             current_data c
-            JOIN filter_data f ON c.source_table = f.source_table_id
+            JOIN (
+              SELECT
+                decode(FILTER ->> 'source_key', 'hex') AS source_key,
+                FILTER ->> 'source_table' AS source_table_id
+              FROM
+                jsonb_array_elements($1::jsonb) AS FILTER
+            ) f ON c.source_table = f.source_table_id
             AND c.source_key = f.source_key
           WHERE
-            c.group_id = $2
+            c.group_id = $2;
         `,
         params: [
           {
@@ -544,7 +561,7 @@ export class PostgresBucketBatch
             value: lookups
           },
           {
-            type: 'int8',
+            type: 'int4',
             value: this.group_id
           }
         ]
@@ -601,7 +618,12 @@ export class PostgresBucketBatch
         await persistedBatch.flush(db);
       }
     }
-    return resumeBatch;
+
+    // Don't return empty batches
+    if (resumeBatch?.batch.length) {
+      return resumeBatch;
+    }
+    return null;
   }
 
   protected async saveOperation(
@@ -618,8 +640,8 @@ export class PostgresBucketBatch
 
     let existingBuckets: CurrentBucket[] = [];
     let newBuckets: CurrentBucket[] = [];
-    let existingLookups: Buffer[] = [];
-    let newLookups: Buffer[] = [];
+    let existingLookups: Buffer<ArrayBuffer>[] = [];
+    let newLookups: Buffer<ArrayBuffer>[] = [];
 
     if (this.options.skip_existing_rows) {
       if (record.tag == storage.SaveOperationTag.INSERT) {
@@ -631,7 +653,7 @@ export class PostgresBucketBatch
           return null;
         }
       } else {
-        throw new Error(`${record.tag} not supported with skipExistingRows: true`);
+        throw new ReplicationAssertionError(`${record.tag} not supported with skipExistingRows: true`);
       }
     }
 
@@ -673,14 +695,14 @@ export class PostgresBucketBatch
       }
     }
 
-    let afterData: Buffer | undefined;
+    let afterData: Buffer<ArrayBuffer> | undefined;
     if (afterId != null && !this.options.store_current_data) {
       afterData = storage.serializeBson({});
     } else if (afterId != null) {
       try {
         afterData = storage.serializeBson(after);
         if (afterData!.byteLength > MAX_ROW_SIZE) {
-          throw new Error(`Row too large: ${afterData?.byteLength}`);
+          throw new ServiceError(ErrorCode.PSYNC_S1002, `Row too large: ${afterData?.byteLength}`);
         }
       } catch (e) {
         // Replace with empty values, equivalent to TOAST values

package/src/storage/batch/PostgresPersistedBatch.ts

@@ -256,33 +256,6 @@ export class PostgresPersistedBatch {
   protected async flushBucketData(db: lib_postgres.WrappedConnection) {
     if (this.bucketDataInserts.length > 0) {
       await db.sql`
-        WITH
-          parsed_data AS (
-            SELECT
-              group_id,
-              bucket_name,
-              source_table,
-              decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
-              table_name,
-              op,
-              row_id,
-              checksum,
-              data,
-              target_op
-            FROM
-              jsonb_to_recordset(${{ type: 'jsonb', value: this.bucketDataInserts }}::jsonb) AS t (
-                group_id integer,
-                bucket_name text,
-                source_table text,
-                source_key text, -- Input as hex string
-                table_name text,
-                op text,
-                row_id text,
-                checksum bigint,
-                data text,
-                target_op bigint
-              )
-          )
         INSERT INTO
           bucket_data (
             group_id,
@@ -303,14 +276,25 @@ export class PostgresPersistedBatch {
           nextval('op_id_sequence'),
           op,
           source_table,
-          source_key, -- Already decoded
+          decode(source_key, 'hex') AS source_key,
           table_name,
           row_id,
           checksum,
           data,
           target_op
         FROM
-          parsed_data;
+          json_to_recordset(${{ type: 'json', value: this.bucketDataInserts }}::json) AS t (
+            group_id integer,
+            bucket_name text,
+            source_table text,
+            source_key text, -- Input as hex string
+            table_name text,
+            op text,
+            row_id text,
+            checksum bigint,
+            data text,
+            target_op bigint
+          );
       `.execute();
     }
   }
@@ -318,23 +302,6 @@ export class PostgresPersistedBatch {
   protected async flushParameterData(db: lib_postgres.WrappedConnection) {
     if (this.parameterDataInserts.length > 0) {
       await db.sql`
-        WITH
-          parsed_data AS (
-            SELECT
-              group_id,
-              source_table,
-              decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
-              decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
-              bucket_parameters
-            FROM
-              jsonb_to_recordset(${{ type: 'jsonb', value: this.parameterDataInserts }}::jsonb) AS t (
-                group_id integer,
-                source_table text,
-                source_key text, -- Input as hex string
-                lookup text, -- Input as hex string
-                bucket_parameters text -- Input as stringified JSON
-              )
-          )
         INSERT INTO
           bucket_parameters (
             group_id,
@@ -346,11 +313,17 @@ export class PostgresPersistedBatch {
         SELECT
           group_id,
           source_table,
-          source_key, -- Already decoded
-          lookup, -- Already decoded
+          decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
+          decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
           bucket_parameters
         FROM
-          parsed_data;
+          json_to_recordset(${{ type: 'json', value: this.parameterDataInserts }}::json) AS t (
+            group_id integer,
+            source_table text,
+            source_key text, -- Input as hex string
+            lookup text, -- Input as hex string
+            bucket_parameters text -- Input as stringified JSON
+          )
       `.execute();
     }
   }
@@ -358,33 +331,6 @@ export class PostgresPersistedBatch {
   protected async flushCurrentData(db: lib_postgres.WrappedConnection) {
     if (this.currentDataInserts.size > 0) {
       await db.sql`
-        WITH
-          parsed_data AS (
-            SELECT
-              group_id,
-              source_table,
-              decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
-              buckets::jsonb AS buckets,
-              decode(data, 'hex') AS data, -- Decode hex to bytea
-              ARRAY(
-                SELECT
-                  decode((value ->> 0)::TEXT, 'hex')
-                FROM
-                  jsonb_array_elements(lookups::jsonb) AS value
-              ) AS lookups -- Decode array of hex strings to bytea[]
-            FROM
-              jsonb_to_recordset(${{
-          type: 'jsonb',
-          value: Array.from(this.currentDataInserts.values())
-        }}::jsonb) AS t (
-                group_id integer,
-                source_table text,
-                source_key text, -- Input as hex string
-                buckets text,
-                data text, -- Input as hex string
-                lookups text -- Input as stringified JSONB array of hex strings
-              )
-          )
         INSERT INTO
           current_data (
             group_id,
@@ -397,12 +343,24 @@ export class PostgresPersistedBatch {
         SELECT
           group_id,
           source_table,
-          source_key, -- Already decoded
-          buckets,
-          data, -- Already decoded
-          lookups -- Already decoded
+          decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
+          buckets::jsonb AS buckets,
+          decode(data, 'hex') AS data, -- Decode hex to bytea
+          array(
+            SELECT
+              decode(element, 'hex')
+            FROM
+              unnest(lookups) AS element
+          ) AS lookups
         FROM
-          parsed_data
+          json_to_recordset(${{ type: 'json', value: Array.from(this.currentDataInserts.values()) }}::json) AS t (
+            group_id integer,
+            source_table text,
+            source_key text, -- Input as hex string
+            buckets text,
+            data text, -- Input as hex string
+            lookups TEXT[] -- Input as stringified JSONB array of hex strings
+          )
         ON CONFLICT (group_id, source_table, source_key) DO UPDATE
         SET
           buckets = EXCLUDED.buckets,

package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts

@@ -1,5 +1,5 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
-import { logger } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 
@@ -44,7 +44,10 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR
     });
     const lockHandle = await manager.acquire();
     if (!lockHandle) {
-      throw new Error(`Sync rules: ${this.id} have been locked by another process for replication.`);
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1003,
+        `Sync rules: ${this.id} have been locked by another process for replication.`
+      );
     }
 
     const interval = setInterval(async () => {

package/src/types/codecs.ts

@@ -1,3 +1,4 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import * as t from 'ts-codec';
 
 export const BIGINT_MAX = BigInt('9223372036854775807');
@@ -98,7 +99,7 @@ export const hexBuffer = t.codec(
       return Buffer.from(encoded);
     }
     if (typeof encoded !== 'string') {
-      throw new Error(`Expected either a Buffer instance or hex encoded buffer string`);
+      throw new ReplicationAssertionError(`Expected either a Buffer instance or hex encoded buffer string`);
     }
     return Buffer.from(encoded, 'hex');
   }
@@ -116,7 +117,7 @@ export const pgwire_number = t.codec(
       return encoded;
     }
     if (typeof encoded !== 'bigint') {
-      throw new Error(`Expected either number or bigint for value`);
+      throw new ReplicationAssertionError(`Expected either number or bigint for value`);
     }
     if (encoded > BigInt(Number.MAX_SAFE_INTEGER) || encoded < BigInt(Number.MIN_SAFE_INTEGER)) {
       throw new RangeError('BigInt value is out of safe integer range for conversion to Number.');

package/tsconfig.json

@@ -5,8 +5,6 @@
     "declarationDir": "dist/@types",
     "tsBuildInfoFile": "dist/.tsbuildinfo",
     "rootDir": "src",
-    "target": "ES2022",
-    "lib": ["ES2022", "esnext.disposable"],
     "skipLibCheck": true
   },
   "include": ["src"],