@powersync/service-module-postgres-storage 0.0.0-dev-20260225093637 → 0.0.0-dev-20260313100403

package/src/storage/PostgresBucketStorageFactory.ts

@@ -1,17 +1,16 @@
-import { GetIntanceOptions, storage, SyncRulesBucketStorage, UpdateSyncRulesOptions } from '@powersync/service-core';
+import { framework, GetIntanceOptions, storage, SyncRulesBucketStorage } from '@powersync/service-core';
 import * as pg_wire from '@powersync/service-jpgwire';
-import * as sync_rules from '@powersync/service-sync-rules';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
 
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import { models, NormalizedPostgresStorageConfig } from '../types/types.js';
 
+import { getStorageApplicationName } from '../utils/application-name.js';
 import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js';
 import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js';
 import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js';
 import { PostgresPersistedSyncRulesContent } from './sync-rules/PostgresPersistedSyncRulesContent.js';
-import { getStorageApplicationName } from '../utils/application-name.js';
 
 export type PostgresBucketStorageOptions = {
   config: NormalizedPostgresStorageConfig;
@@ -83,15 +82,27 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
 
     const sizes = await this.db.sql`
       SELECT
-        pg_total_relation_size('current_data') AS current_size_bytes,
+        COALESCE(
+          pg_total_relation_size(to_regclass('current_data')),
+          0
+        ) AS v1_current_size_bytes,
+        COALESCE(
+          pg_total_relation_size(to_regclass('v3_current_data')),
+          0
+        ) AS v3_current_size_bytes,
         pg_total_relation_size('bucket_parameters') AS parameter_size_bytes,
         pg_total_relation_size('bucket_data') AS operations_size_bytes;
-    `.first<{ current_size_bytes: bigint; parameter_size_bytes: bigint; operations_size_bytes: bigint }>();
+    `.first<{
+      v1_current_size_bytes: bigint;
+      v3_current_size_bytes: bigint;
+      parameter_size_bytes: bigint;
+      operations_size_bytes: bigint;
+    }>();
 
     return {
       operations_size_bytes: Number(sizes!.operations_size_bytes),
       parameters_size_bytes: Number(sizes!.parameter_size_bytes),
-      replication_size_bytes: Number(sizes!.current_size_bytes)
+      replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes)
     };
   }
 
@@ -143,6 +154,14 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
 
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedSyncRulesContent> {
     const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+    const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
+    if (storageConfig == null) {
+      throw new framework.ServiceError(
+        framework.ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${storageVersion}`
+      );
+    }
+    await this.initializeStorageVersion(storageConfig);
     return this.db.transaction(async (db) => {
       await db.sql`
         UPDATE sync_rules
@@ -159,7 +178,14 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
               nextval('sync_rules_id_sequence') AS id
           )
         INSERT INTO
-          sync_rules (id, content, state, slot_name, storage_version)
+          sync_rules (
+            id,
+            content,
+            sync_plan,
+            state,
+            slot_name,
+            storage_version
+          )
         VALUES
           (
             (
@@ -169,6 +195,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
                 next_id
             ),
             ${{ type: 'varchar', value: options.config.yaml }},
+            ${{ type: 'json', value: options.config.plan }},
             ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
             CONCAT(
               ${{ type: 'varchar', value: this.slot_name_prefix }},
@@ -195,6 +222,34 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
     });
   }
 
+  /**
+   * Lazy-initializes storage-version-specific structures, if needed.
+   */
+  private async initializeStorageVersion(storageConfig: storage.StorageVersionConfig) {
+    if (!storageConfig.softDeleteCurrentData) {
+      return;
+    }
+
+    await this.db.sql`
+      CREATE TABLE IF NOT EXISTS v3_current_data (
+        group_id integer NOT NULL,
+        source_table TEXT NOT NULL,
+        source_key bytea NOT NULL,
+        CONSTRAINT unique_v3_current_data_id PRIMARY KEY (group_id, source_table, source_key),
+        buckets jsonb NOT NULL,
+        data bytea NOT NULL,
+        lookups bytea[] NOT NULL,
+        pending_delete BIGINT NULL
+      )
+    `.execute();
+
+    await this.db.sql`
+      CREATE INDEX IF NOT EXISTS v3_current_data_pending_deletes ON v3_current_data (group_id, pending_delete)
+      WHERE
+        pending_delete IS NOT NULL
+    `.execute();
+  }
+
   async restartReplication(sync_rules_group_id: number): Promise<void> {
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();

package/src/storage/PostgresCompactor.ts

@@ -51,20 +51,20 @@ export class PostgresCompactor {
   private moveBatchLimit: number;
   private moveBatchQueryLimit: number;
   private clearBatchLimit: number;
-  private maxOpId: InternalOpId | undefined;
+  private maxOpId: InternalOpId;
   private buckets: string[] | undefined;
 
   constructor(
     private db: lib_postgres.DatabaseClient,
     private group_id: number,
-    options?: PostgresCompactOptions
+    options: PostgresCompactOptions
   ) {
-    this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
-    this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
-    this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
-    this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
-    this.maxOpId = options?.maxOpId;
-    this.buckets = options?.compactBuckets;
+    this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
+    this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
+    this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
+    this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
+    this.maxOpId = options.maxOpId ?? 0n;
+    this.buckets = options.compactBuckets;
   }
 
   /**
@@ -240,6 +240,15 @@ export class PostgresCompactor {
     }
   }
 
+  /**
+   * Expose the internal clearBucket() method to tests.
+   *
+   * @deprecated Only for tests
+   */
+  clearBucketForTests(bucket: string, op: InternalOpId) {
+    return this.clearBucket(bucket, op);
+  }
+
   /**
    * Perform a CLEAR compact for a bucket.
    *

package/src/storage/PostgresReportStorage.ts

@@ -48,13 +48,13 @@ export class PostgresReportStorage implements storage.ReportStorage {
   ): event_types.ClientConnectionReportResponse {
     if (!result) {
       return {
-        users: 0,
-        sdks: []
+        total_users: 0,
+        sdk_breakdown: []
       };
     }
     return {
-      users: Number(result.users),
-      sdks: result.sdks?.data || []
+      total_users: Number(result.users),
+      sdk_breakdown: result.sdks?.data || []
     };
   }
   private async listConnectionsQuery() {
@@ -234,12 +234,12 @@ export class PostgresReportStorage implements storage.ReportStorage {
         AND connected_at = ${{ type: 1184, value: connectIsoString }}
     `.execute();
   }
-  async getConnectedClients(): Promise<event_types.ClientConnectionReportResponse> {
+  async getCurrentConnections(): Promise<event_types.ClientConnectionReportResponse> {
     const result = await this.listConnectionsQuery();
     return this.mapListCurrentConnectionsResponse(result);
   }
 
-  async getClientConnectionReports(
+  async getClientConnectionsSummary(
     data: event_types.ClientConnectionReportRequest
   ): Promise<event_types.ClientConnectionReportResponse> {
     const { start, end } = data;
@@ -289,7 +289,7 @@ export class PostgresReportStorage implements storage.ReportStorage {
     return this.mapListCurrentConnectionsResponse(result);
   }
 
-  async getGeneralClientConnectionAnalytics(
+  async getClientSessions(
     data: event_types.ClientConnectionAnalyticsRequest
   ): Promise<event_types.PaginatedResponse<event_types.ClientConnection>> {
     const limit = data.limit || 100;

package/src/storage/PostgresSyncRulesStorage.ts

@@ -14,6 +14,7 @@ import {
   PopulateChecksumCacheResults,
   ReplicationCheckpoint,
   storage,
+  StorageVersionConfig,
   utils,
   WatchWriteCheckpointOptions
 } from '@powersync/service-core';
@@ -35,6 +36,7 @@ import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js';
 import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js';
 import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
 import { PostgresCompactor } from './PostgresCompactor.js';
+import { PostgresCurrentDataStore } from './current-data-store.js';
 
 export type PostgresSyncRulesStorageOptions = {
   factory: PostgresBucketStorageFactory;
@@ -52,11 +54,13 @@ export class PostgresSyncRulesStorage
   public readonly sync_rules: storage.PersistedSyncRulesContent;
   public readonly slot_name: string;
   public readonly factory: PostgresBucketStorageFactory;
+  public readonly storageConfig: StorageVersionConfig;
 
   private sharedIterator = new BroadcastIterable((signal) => this.watchActiveCheckpoint(signal));
 
   protected db: lib_postgres.DatabaseClient;
   protected writeCheckpointAPI: PostgresWriteCheckpointAPI;
+  private readonly currentDataStore: PostgresCurrentDataStore;
 
   //   TODO we might be able to share this in an abstract class
   private parsedSyncRulesCache:
@@ -71,6 +75,8 @@ export class PostgresSyncRulesStorage
     this.sync_rules = options.sync_rules;
     this.slot_name = options.sync_rules.slot_name;
     this.factory = options.factory;
+    this.storageConfig = options.sync_rules.getStorageConfig();
+    this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
 
     this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
       db: this.db,
@@ -121,8 +127,18 @@ export class PostgresSyncRulesStorage
     `.execute();
   }
 
-  compact(options?: storage.CompactOptions): Promise<void> {
-    return new PostgresCompactor(this.db, this.group_id, options).compact();
+  async compact(options?: storage.CompactOptions): Promise<void> {
+    let maxOpId = options?.maxOpId;
+    if (maxOpId == null) {
+      const checkpoint = await this.getCheckpoint();
+      // Note: If there is no active checkpoint, this will be 0, in which case no compacting is performed
+      maxOpId = checkpoint.checkpoint;
+    }
+
+    return new PostgresCompactor(this.db, this.group_id, {
+      ...options,
+      maxOpId
+    }).compact();
   }
 
   async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
@@ -327,10 +343,7 @@ export class PostgresSyncRulesStorage
     });
   }
 
-  async startBatch(
-    options: storage.StartBatchOptions,
-    callback: (batch: storage.BucketStorageBatch) => Promise<void>
-  ): Promise<storage.FlushedResult | null> {
+  async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
     const syncRules = await this.db.sql`
       SELECT
         last_checkpoint_lsn,
@@ -347,7 +360,7 @@ export class PostgresSyncRulesStorage
 
     const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null;
 
-    const batch = new PostgresBucketBatch({
+    const writer = new PostgresBucketBatch({
       logger: options.logger ?? framework.logger,
       db: this.db,
       sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(),
@@ -355,22 +368,28 @@ export class PostgresSyncRulesStorage
       slot_name: this.slot_name,
       last_checkpoint_lsn: checkpoint_lsn,
       keep_alive_op: syncRules?.keepalive_op,
-      no_checkpoint_before_lsn: syncRules?.no_checkpoint_before ?? options.zeroLSN,
       resumeFromLsn: maxLsn(syncRules?.snapshot_lsn, checkpoint_lsn),
       store_current_data: options.storeCurrentData,
       skip_existing_rows: options.skipExistingRows ?? false,
       batch_limits: this.options.batchLimits,
-      markRecordUnavailable: options.markRecordUnavailable
+      markRecordUnavailable: options.markRecordUnavailable,
+      storageConfig: this.storageConfig
     });
-    this.iterateListeners((cb) => cb.batchStarted?.(batch));
-
-    await callback(batch);
-    await batch.flush();
-    if (batch.last_flushed_op != null) {
-      return { flushed_op: batch.last_flushed_op };
-    } else {
-      return null;
-    }
+    this.iterateListeners((cb) => cb.batchStarted?.(writer));
+    return writer;
+  }
+
+  /**
+   * @deprecated Use `createWriter()` with `await using` instead.
+   */
+  async startBatch(
+    options: storage.CreateWriterOptions,
+    callback: (batch: storage.BucketStorageBatch) => Promise<void>
+  ): Promise<storage.FlushedResult | null> {
+    await using writer = await this.createWriter(options);
+    await callback(writer);
+    await writer.flush();
+    return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null;
   }
 
   async getParameterSets(
@@ -415,10 +434,10 @@ export class PostgresSyncRulesStorage
 
   async *getBucketDataBatch(
     checkpoint: InternalOpId,
-    dataBuckets: Map<string, InternalOpId>,
+    dataBuckets: storage.BucketDataRequest[],
     options?: storage.BucketDataBatchOptions
   ): AsyncIterable<storage.SyncBucketDataChunk> {
-    if (dataBuckets.size == 0) {
+    if (dataBuckets.length == 0) {
       return;
     }
 
@@ -430,10 +449,8 @@ export class PostgresSyncRulesStorage
     // not match up with chunks.
 
     const end = checkpoint ?? BIGINT_MAX;
-    const filters = Array.from(dataBuckets.entries()).map(([name, start]) => ({
-      bucket_name: name,
-      start: start
-    }));
+    const filters = dataBuckets.map((request) => ({ bucket_name: request.bucket, start: request.start }));
+    const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
 
     const batchRowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
     const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
@@ -533,7 +550,7 @@ export class PostgresSyncRulesStorage
           }
 
           if (start == null) {
-            const startOpId = dataBuckets.get(bucket_name);
+            const startOpId = startOpByBucket.get(bucket_name);
             if (startOpId == null) {
               throw new framework.ServiceAssertionError(`data for unexpected bucket: ${bucket_name}`);
             }
@@ -588,7 +605,10 @@ export class PostgresSyncRulesStorage
     }
   }
 
-  async getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap> {
+  async getChecksums(
+    checkpoint: utils.InternalOpId,
+    buckets: storage.BucketChecksumRequest[]
+  ): Promise<utils.ChecksumMap> {
     return this.checksumCache.getChecksumMap(checkpoint, buckets);
   }
 
@@ -662,11 +682,7 @@ export class PostgresSyncRulesStorage
         group_id = ${{ type: 'int4', value: this.group_id }}
     `.execute();
 
-    await this.db.sql`
-      DELETE FROM current_data
-      WHERE
-        group_id = ${{ type: 'int4', value: this.group_id }}
-    `.execute();
+    await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.group_id });
 
     await this.db.sql`
       DELETE FROM source_tables

package/src/storage/batch/OperationBatch.ts

@@ -5,6 +5,7 @@
 
 import { storage, utils } from '@powersync/service-core';
 import { RequiredOperationBatchLimits } from '../../types/types.js';
+import { postgresTableId } from '../table-id.js';
 
 /**
  * Batch of input operations.
@@ -89,13 +90,13 @@ export class RecordOperation {
 /**
  * In-memory cache key - must not be persisted.
  */
-export function cacheKey(sourceTableId: string, id: storage.ReplicaId) {
+export function cacheKey(sourceTableId: storage.SourceTableId, id: storage.ReplicaId) {
   return encodedCacheKey(sourceTableId, storage.serializeReplicaId(id));
 }
 
 /**
  * Calculates a cache key for a stored ReplicaId. This is usually stored as a bytea/Buffer.
  */
-export function encodedCacheKey(sourceTableId: string, storedKey: Buffer) {
-  return `${sourceTableId}.${storedKey.toString('base64')}`;
+export function encodedCacheKey(sourceTableId: storage.SourceTableId, storedKey: Buffer) {
+  return `${postgresTableId(sourceTableId)}.${storedKey.toString('base64')}`;
 }