@powersync/service-module-postgres-storage 0.4.2 → 0.5.1

package/src/storage/PostgresSyncRulesStorage.ts

@@ -5,6 +5,8 @@ import {
   CHECKPOINT_INVALIDATE_ALL,
   CheckpointChanges,
   GetCheckpointChangesOptions,
+  InternalOpId,
+  internalToExternalOpId,
   LastValueSink,
   storage,
   utils,
@@ -149,7 +151,7 @@ export class PostgresSyncRulesStorage
       .first();
 
     return {
-      checkpoint: utils.timestampToOpId(checkpointRow?.last_checkpoint ?? 0n),
+      checkpoint: checkpointRow?.last_checkpoint ?? 0n,
       lsn: checkpointRow?.last_checkpoint_lsn ?? null
     };
   }
@@ -344,14 +346,14 @@ export class PostgresSyncRulesStorage
     await callback(batch);
     await batch.flush();
     if (batch.last_flushed_op) {
-      return { flushed_op: String(batch.last_flushed_op) };
+      return { flushed_op: batch.last_flushed_op };
     } else {
       return null;
     }
   }
 
   async getParameterSets(
-    checkpoint: utils.OpId,
+    checkpoint: utils.InternalOpId,
     lookups: sync_rules.SqliteJsonValue[][]
   ): Promise<sync_rules.SqliteJsonRow[]> {
     const rows = await this.db.sql`
@@ -374,7 +376,7 @@ export class PostgresSyncRulesStorage
         value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex'))
       }}) AS FILTER
         )
-        AND id <= ${{ type: 'int8', value: BigInt(checkpoint) }}
+        AND id <= ${{ type: 'int8', value: checkpoint }}
       ORDER BY
         lookup,
         source_table,
@@ -391,8 +393,8 @@ export class PostgresSyncRulesStorage
   }
 
   async *getBucketDataBatch(
-    checkpoint: utils.OpId,
-    dataBuckets: Map<string, string>,
+    checkpoint: InternalOpId,
+    dataBuckets: Map<string, InternalOpId>,
     options?: storage.BucketDataBatchOptions
   ): AsyncIterable<storage.SyncBucketDataBatch> {
     if (dataBuckets.size == 0) {
@@ -410,7 +412,7 @@ export class PostgresSyncRulesStorage
 
     let batchSize = 0;
     let currentBatch: utils.SyncBucketData | null = null;
-    let targetOp: bigint | null = null;
+    let targetOp: InternalOpId | null = null;
     let rowCount = 0;
 
     /**
@@ -503,9 +505,12 @@ export class PostgresSyncRulesStorage
             }
           }
 
-          start ??= dataBuckets.get(bucket_name);
           if (start == null) {
-            throw new ReplicationAssertionError(`data for unexpected bucket: ${bucket_name}`);
+            const startOpId = dataBuckets.get(bucket_name);
+            if (startOpId == null) {
+              throw new framework.ServiceAssertionError(`data for unexpected bucket: ${bucket_name}`);
+            }
+            start = internalToExternalOpId(startOpId);
           }
           currentBatch = {
             bucket: bucket_name,
@@ -549,7 +554,7 @@ export class PostgresSyncRulesStorage
     }
   }
 
-  async getChecksums(checkpoint: utils.OpId, buckets: string[]): Promise<utils.ChecksumMap> {
+  async getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap> {
     return this.checksumCache.getChecksumMap(checkpoint, buckets);
   }
 
@@ -672,8 +677,9 @@ export class PostgresSyncRulesStorage
     }
 
     const rangedBatch = batch.map((b) => ({
-      ...b,
-      start: b.start ?? 0
+      bucket: b.bucket,
+      start: String(b.start ?? 0n),
+      end: String(b.end)
     }));
 
     const results = await this.db.sql`
@@ -745,7 +751,7 @@ export class PostgresSyncRulesStorage
   }
 
   async *watchWriteCheckpoint(options: WatchWriteCheckpointOptions): AsyncIterable<storage.StorageCheckpointUpdate> {
-    let lastCheckpoint: utils.OpId | null = null;
+    let lastCheckpoint: utils.InternalOpId | null = null;
     let lastWriteCheckpoint: bigint | null = null;
 
     const { signal, user_id } = options;
@@ -852,7 +858,7 @@ export class PostgresSyncRulesStorage
 
   private makeActiveCheckpoint(row: models.ActiveCheckpointDecoded | null) {
     return {
-      checkpoint: utils.timestampToOpId(row?.last_checkpoint ?? 0n),
+      checkpoint: row?.last_checkpoint ?? 0n,
       lsn: row?.last_checkpoint_lsn ?? null
     } satisfies storage.ReplicationCheckpoint;
   }

package/src/storage/batch/PostgresBucketBatch.ts

@@ -9,7 +9,7 @@ import {
   ServiceAssertionError,
   ServiceError
 } from '@powersync/lib-services-framework';
-import { storage, utils } from '@powersync/service-core';
+import { InternalOpId, storage, utils } from '@powersync/service-core';
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import * as t from 'ts-codec';
@@ -29,7 +29,7 @@ export interface PostgresBucketBatchOptions {
   last_checkpoint_lsn: string | null;
   no_checkpoint_before_lsn: string;
   store_current_data: boolean;
-  keep_alive_op?: bigint | null;
+  keep_alive_op?: InternalOpId | null;
   /**
    * Set to true for initial replication.
    */
@@ -54,14 +54,14 @@ export class PostgresBucketBatch
   extends BaseObserver<storage.BucketBatchStorageListener>
   implements storage.BucketStorageBatch
 {
-  public last_flushed_op: bigint | null = null;
+  public last_flushed_op: InternalOpId | null = null;
 
   protected db: lib_postgres.DatabaseClient;
   protected group_id: number;
   protected last_checkpoint_lsn: string | null;
   protected no_checkpoint_before_lsn: string;
 
-  protected persisted_op: bigint | null;
+  protected persisted_op: InternalOpId | null;
 
   protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[];
   protected readonly sync_rules: sync_rules.SqlSyncRules;
@@ -132,18 +132,19 @@ export class PostgresBucketBatch
   async truncate(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null> {
     await this.flush();
 
-    let last_op: bigint | null = null;
+    let last_op: InternalOpId | null = null;
     for (let table of sourceTables) {
       last_op = await this.truncateSingle(table);
     }
 
     if (last_op) {
       this.persisted_op = last_op;
+      return {
+        flushed_op: last_op
+      };
+    } else {
+      return null;
     }
-
-    return {
-      flushed_op: String(last_op!)
-    };
   }
 
   protected async truncateSingle(sourceTable: storage.SourceTable) {
@@ -279,7 +280,7 @@ export class PostgresBucketBatch
 
     this.persisted_op = lastOp;
     this.last_flushed_op = lastOp;
-    return { flushed_op: String(lastOp) };
+    return { flushed_op: lastOp };
   }
 
   async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<boolean> {

package/src/types/types.ts

@@ -53,6 +53,7 @@ export type RequiredOperationBatchLimits = Required<OperationBatchLimits>;
 
 export type NormalizedPostgresStorageConfig = pg_wire.NormalizedConnectionConfig & {
   batch_limits: RequiredOperationBatchLimits;
+  max_pool_size: number;
 };
 
 export const normalizePostgresStorageConfig = (

package/src/utils/bucket-data.ts

@@ -5,7 +5,7 @@ import { replicaIdToSubkey } from './bson.js';
 export const mapOpEntry = (entry: models.BucketDataDecoded) => {
   if (entry.op == models.OpType.PUT || entry.op == models.OpType.REMOVE) {
     return {
-      op_id: utils.timestampToOpId(entry.op_id),
+      op_id: utils.internalToExternalOpId(entry.op_id),
       op: entry.op,
       object_type: entry.table_name ?? undefined,
       object_id: entry.row_id ?? undefined,
@@ -17,7 +17,7 @@ export const mapOpEntry = (entry: models.BucketDataDecoded) => {
     // MOVE, CLEAR
 
     return {
-      op_id: utils.timestampToOpId(entry.op_id),
+      op_id: utils.internalToExternalOpId(entry.op_id),
       op: entry.op,
       checksum: Number(entry.checksum)
     };

package/test/src/storage.test.ts

@@ -81,7 +81,7 @@ describe('Postgres Sync Bucket Storage', () => {
     const options: storage.BucketDataBatchOptions = {};
 
     const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), options)
     );
     expect(test_utils.getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -93,7 +93,7 @@ describe('Postgres Sync Bucket Storage', () => {
     });
 
     const batch2 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch1[0].batch.next_after)]]), options)
     );
     expect(test_utils.getBatchData(batch2)).toEqual([
       { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
@@ -105,7 +105,7 @@ describe('Postgres Sync Bucket Storage', () => {
     });
 
     const batch3 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch2[0].batch.next_after)]]), options)
     );
     expect(test_utils.getBatchData(batch3)).toEqual([
       { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
@@ -117,7 +117,7 @@ describe('Postgres Sync Bucket Storage', () => {
     });
 
     const batch4 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch3[0].batch.next_after]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch3[0].batch.next_after)]]), options)
     );
     expect(test_utils.getBatchData(batch4)).toEqual([
       { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }