@powersync/service-module-postgres-storage 0.12.0 → 0.13.1

package/src/storage/PostgresSyncRulesStorage.ts

@@ -14,6 +14,7 @@ import {
   PopulateChecksumCacheResults,
   ReplicationCheckpoint,
   storage,
+  StorageVersionConfig,
   utils,
   WatchWriteCheckpointOptions
 } from '@powersync/service-core';
@@ -35,6 +36,7 @@ import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js';
 import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js';
 import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
 import { PostgresCompactor } from './PostgresCompactor.js';
+import { PostgresCurrentDataStore } from './current-data-store.js';
 
 export type PostgresSyncRulesStorageOptions = {
   factory: PostgresBucketStorageFactory;
@@ -52,11 +54,13 @@ export class PostgresSyncRulesStorage
   public readonly sync_rules: storage.PersistedSyncRulesContent;
   public readonly slot_name: string;
   public readonly factory: PostgresBucketStorageFactory;
+  public readonly storageConfig: StorageVersionConfig;
 
   private sharedIterator = new BroadcastIterable((signal) => this.watchActiveCheckpoint(signal));
 
   protected db: lib_postgres.DatabaseClient;
   protected writeCheckpointAPI: PostgresWriteCheckpointAPI;
+  private readonly currentDataStore: PostgresCurrentDataStore;
 
   //   TODO we might be able to share this in an abstract class
   private parsedSyncRulesCache:
@@ -71,6 +75,8 @@ export class PostgresSyncRulesStorage
     this.sync_rules = options.sync_rules;
     this.slot_name = options.sync_rules.slot_name;
     this.factory = options.factory;
+    this.storageConfig = options.sync_rules.getStorageConfig();
+    this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
 
     this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
       db: this.db,
@@ -121,8 +127,18 @@ export class PostgresSyncRulesStorage
     `.execute();
   }
 
-  compact(options?: storage.CompactOptions): Promise<void> {
-    return new PostgresCompactor(this.db, this.group_id, options).compact();
+  async compact(options?: storage.CompactOptions): Promise<void> {
+    let maxOpId = options?.maxOpId;
+    if (maxOpId == null) {
+      const checkpoint = await this.getCheckpoint();
+      // Note: If there is no active checkpoint, this will be 0, in which case no compacting is performed
+      maxOpId = checkpoint.checkpoint;
+    }
+
+    return new PostgresCompactor(this.db, this.group_id, {
+      ...options,
+      maxOpId
+    }).compact();
   }
 
   async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
@@ -327,10 +343,7 @@ export class PostgresSyncRulesStorage
     });
   }
 
-  async startBatch(
-    options: storage.StartBatchOptions,
-    callback: (batch: storage.BucketStorageBatch) => Promise<void>
-  ): Promise<storage.FlushedResult | null> {
+  async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
     const syncRules = await this.db.sql`
       SELECT
         last_checkpoint_lsn,
@@ -347,7 +360,7 @@ export class PostgresSyncRulesStorage
 
     const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null;
 
-    const batch = new PostgresBucketBatch({
+    const writer = new PostgresBucketBatch({
       logger: options.logger ?? framework.logger,
       db: this.db,
       sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(),
@@ -355,22 +368,28 @@ export class PostgresSyncRulesStorage
       slot_name: this.slot_name,
       last_checkpoint_lsn: checkpoint_lsn,
       keep_alive_op: syncRules?.keepalive_op,
-      no_checkpoint_before_lsn: syncRules?.no_checkpoint_before ?? options.zeroLSN,
       resumeFromLsn: maxLsn(syncRules?.snapshot_lsn, checkpoint_lsn),
       store_current_data: options.storeCurrentData,
       skip_existing_rows: options.skipExistingRows ?? false,
       batch_limits: this.options.batchLimits,
-      markRecordUnavailable: options.markRecordUnavailable
+      markRecordUnavailable: options.markRecordUnavailable,
+      storageConfig: this.storageConfig
     });
-    this.iterateListeners((cb) => cb.batchStarted?.(batch));
-
-    await callback(batch);
-    await batch.flush();
-    if (batch.last_flushed_op != null) {
-      return { flushed_op: batch.last_flushed_op };
-    } else {
-      return null;
-    }
+    this.iterateListeners((cb) => cb.batchStarted?.(writer));
+    return writer;
+  }
+
+  /**
+   * @deprecated Use `createWriter()` with `await using` instead.
+   */
+  async startBatch(
+    options: storage.CreateWriterOptions,
+    callback: (batch: storage.BucketStorageBatch) => Promise<void>
+  ): Promise<storage.FlushedResult | null> {
+    await using writer = await this.createWriter(options);
+    await callback(writer);
+    await writer.flush();
+    return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null;
   }
 
   async getParameterSets(
@@ -415,10 +434,10 @@ export class PostgresSyncRulesStorage
 
   async *getBucketDataBatch(
     checkpoint: InternalOpId,
-    dataBuckets: Map<string, InternalOpId>,
+    dataBuckets: storage.BucketDataRequest[],
     options?: storage.BucketDataBatchOptions
   ): AsyncIterable<storage.SyncBucketDataChunk> {
-    if (dataBuckets.size == 0) {
+    if (dataBuckets.length == 0) {
       return;
     }
 
@@ -430,10 +449,8 @@ export class PostgresSyncRulesStorage
     // not match up with chunks.
 
     const end = checkpoint ?? BIGINT_MAX;
-    const filters = Array.from(dataBuckets.entries()).map(([name, start]) => ({
-      bucket_name: name,
-      start: start
-    }));
+    const filters = dataBuckets.map((request) => ({ bucket_name: request.bucket, start: request.start }));
+    const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
 
     const batchRowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
     const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
@@ -533,7 +550,7 @@ export class PostgresSyncRulesStorage
           }
 
           if (start == null) {
-            const startOpId = dataBuckets.get(bucket_name);
+            const startOpId = startOpByBucket.get(bucket_name);
             if (startOpId == null) {
               throw new framework.ServiceAssertionError(`data for unexpected bucket: ${bucket_name}`);
             }
@@ -588,7 +605,10 @@ export class PostgresSyncRulesStorage
     }
   }
 
-  async getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap> {
+  async getChecksums(
+    checkpoint: utils.InternalOpId,
+    buckets: storage.BucketChecksumRequest[]
+  ): Promise<utils.ChecksumMap> {
     return this.checksumCache.getChecksumMap(checkpoint, buckets);
   }
 
@@ -662,11 +682,7 @@ export class PostgresSyncRulesStorage
         group_id = ${{ type: 'int4', value: this.group_id }}
     `.execute();
 
-    await this.db.sql`
-      DELETE FROM current_data
-      WHERE
-        group_id = ${{ type: 'int4', value: this.group_id }}
-    `.execute();
+    await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.group_id });
 
     await this.db.sql`
       DELETE FROM source_tables

package/src/storage/batch/OperationBatch.ts

@@ -5,6 +5,7 @@
 
 import { storage, utils } from '@powersync/service-core';
 import { RequiredOperationBatchLimits } from '../../types/types.js';
+import { postgresTableId } from '../table-id.js';
 
 /**
  * Batch of input operations.
@@ -89,13 +90,13 @@ export class RecordOperation {
 /**
  * In-memory cache key - must not be persisted.
  */
-export function cacheKey(sourceTableId: string, id: storage.ReplicaId) {
+export function cacheKey(sourceTableId: storage.SourceTableId, id: storage.ReplicaId) {
   return encodedCacheKey(sourceTableId, storage.serializeReplicaId(id));
 }
 
 /**
  * Calculates a cache key for a stored ReplicaId. This is usually stored as a bytea/Buffer.
  */
-export function encodedCacheKey(sourceTableId: string, storedKey: Buffer) {
-  return `${sourceTableId}.${storedKey.toString('base64')}`;
+export function encodedCacheKey(sourceTableId: storage.SourceTableId, storedKey: Buffer) {
+  return `${postgresTableId(sourceTableId)}.${storedKey.toString('base64')}`;
 }