@powersync/service-module-postgres-storage 0.11.2 → 0.13.0

package/dist/utils/test-utils.js

@@ -55,6 +55,8 @@ import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'
 import { normalizePostgresStorageConfig } from '../types/types.js';
 import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
 import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
+import { logger as defaultLogger, createLogger, transports } from '@powersync/lib-services-framework';
+import { truncateTables } from './db.js';
 export function postgresTestSetup(factoryOptions) {
     const BASE_CONFIG = {
         type: 'postgresql',
@@ -62,7 +64,7 @@ export function postgresTestSetup(factoryOptions) {
         sslmode: 'disable'
     };
     const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG);
-    const migrate = async (direction) => {
+    const runMigrations = async (options) => {
         const env_1 = { stack: [], error: void 0, hasError: false };
         try {
             const migrationManager = __addDisposableResource(env_1, new framework.MigrationManager(), true);
@@ -71,18 +73,28 @@ export function postgresTestSetup(factoryOptions) {
                 : new PostgresMigrationAgent(BASE_CONFIG), true);
             migrationManager.registerMigrationAgent(migrationAgent);
             const mockServiceContext = { configuration: { storage: BASE_CONFIG } };
-            await migrationManager.migrate({
-                direction: framework.migrations.Direction.Down,
-                migrationContext: {
-                    service_context: mockServiceContext
-                }
+            // Migration logs can get really verbose in tests, so only log warnings and up.
+            const logger = createLogger({
+                level: 'warn',
+                format: defaultLogger.format,
+                transports: [new transports.Console()]
             });
-            if (direction == framework.migrations.Direction.Up) {
+            if (options.down) {
+                await migrationManager.migrate({
+                    direction: framework.migrations.Direction.Down,
+                    migrationContext: {
+                        service_context: mockServiceContext
+                    },
+                    logger
+                });
+            }
+            if (options.up) {
                 await migrationManager.migrate({
                     direction: framework.migrations.Direction.Up,
                     migrationContext: {
                         service_context: mockServiceContext
-                    }
+                    },
+                    logger
                 });
             }
         }
@@ -96,11 +108,37 @@ export function postgresTestSetup(factoryOptions) {
                 await result_1;
         }
     };
+    const migrate = async (direction) => {
+        await runMigrations({
+            down: true,
+            up: direction == framework.migrations.Direction.Up
+        });
+    };
+    const clearStorage = async () => {
+        const env_2 = { stack: [], error: void 0, hasError: false };
+        try {
+            await runMigrations({ down: false, up: true });
+            const storageFactory = __addDisposableResource(env_2, new PostgresBucketStorageFactory({
+                config: TEST_CONNECTION_OPTIONS,
+                slot_name_prefix: 'test_'
+            }), true);
+            await truncateTables(storageFactory.db);
+        }
+        catch (e_2) {
+            env_2.error = e_2;
+            env_2.hasError = true;
+        }
+        finally {
+            const result_2 = __disposeResources(env_2);
+            if (result_2)
+                await result_2;
+        }
+    };
     return {
         reportFactory: async (options) => {
             try {
                 if (!options?.doNotClear) {
-                    await migrate(framework.migrations.Direction.Up);
+                    await clearStorage();
                 }
                 return new PostgresReportStorage({
                     config: TEST_CONNECTION_OPTIONS
@@ -115,7 +153,7 @@ export function postgresTestSetup(factoryOptions) {
         factory: async (options) => {
             try {
                 if (!options?.doNotClear) {
-                    await migrate(framework.migrations.Direction.Up);
+                    await clearStorage();
                 }
                 return new PostgresBucketStorageFactory({
                     config: TEST_CONNECTION_OPTIONS,
@@ -128,10 +166,8 @@ export function postgresTestSetup(factoryOptions) {
                 throw ex;
             }
         },
-        migrate
+        migrate,
+        tableIdStrings: true
     };
 }
-export function postgresTestStorageFactoryGenerator(factoryOptions) {
-    return postgresTestSetup(factoryOptions).factory;
-}
 //# sourceMappingURL=test-utils.js.map

package/dist/utils/test-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,8BAA8B,EAAgC,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAW1F,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE5E,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;;;YAClE,MAAY,gBAAgB,kCAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,OAAA,CAAC;YAC3F,MAAY,cAAc,kCAAG,cAAc,CAAC,cAAc;gBACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;gBAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,OAAA,CAAC;YAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;YAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;YAEpG,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;aACF,CAAC,CAAC;YAEH,IAAI,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBACnD,MAAM,gBAAgB,CAAC,OAAO,CAAC;oBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;oBAC5C,gBAAgB,EAAE;wBAChB,eAAe,EAAE,kBAAkB;qBACpC;iBACF,CAAC,CAAC;YACL,CAAC;;;;;;;;;;;KACF,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,gBAAgB,EAAE,OAAO;iBAC1B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,cAA0C;IAC5F,OAAO,iBAAiB,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC;AACnD,CAAC"}
+{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,8BAA8B,EAAgC,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAWzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;;;YACtE,MAAY,gBAAgB,kCAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,OAAA,CAAC;YAC3F,MAAY,cAAc,kCAAG,cAAc,CAAC,cAAc;gBACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;gBAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,OAAA,CAAC;YAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;YAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;YAEpG,+EAA+E;YAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;gBAC1B,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,aAAa,CAAC,MAAM;gBAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;aACvC,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;oBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;oBAC9C,gBAAgB,EAAE;wBAChB,eAAe,EAAE,kBAAkB;qBACpC;oBACD,MAAM;iBACP,CAAC,CAAC;YACL,CAAC;YAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;oBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;oBAC5C,gBAAgB,EAAE;wBAChB,eAAe,EAAE,kBAAkB;qBACpC;oBACD,MAAM;iBACP,CAAC,CAAC;YACL,CAAC;;;;;;;;;;;KACF,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;;YAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;YAE/C,MAAY,cAAc,kCAAG,IAAI,4BAA4B,CAAC;gBAC5D,MAAM,EAAE,uBAAuB;gBAC/B,gBAAgB,EAAE,OAAO;aAC1B,CAAC,OAAA,CAAC;YACH,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;;;;;;;;;;;KACzC,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,gBAAgB,EAAE,OAAO;iBAC1B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;QACP,cAAc,EAAE,IAAI;KACrB,CAAC;AACJ,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-postgres-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/@types/index.d.ts",
-  "version": "0.11.2",
+  "version": "0.13.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-ALv2",
   "type": "module",
@@ -29,22 +29,22 @@
     "p-defer": "^4.0.1",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-postgres": "0.4.21",
-    "@powersync/lib-services-framework": "0.8.2",
-    "@powersync/service-core": "1.19.2",
-    "@powersync/service-types": "0.14.0",
-    "@powersync/service-jpgwire": "0.21.12",
+    "@powersync/lib-service-postgres": "0.4.23",
+    "@powersync/lib-services-framework": "0.9.0",
+    "@powersync/service-core": "1.20.1",
+    "@powersync/service-types": "0.15.0",
+    "@powersync/service-jpgwire": "0.21.14",
     "@powersync/service-jsonbig": "0.17.12",
-    "@powersync/service-sync-rules": "0.31.1"
+    "@powersync/service-sync-rules": "0.33.0"
   },
   "devDependencies": {
     "typescript": "^5.7.3",
-    "@powersync/service-core-tests": "0.13.2"
+    "@powersync/service-core-tests": "0.15.0"
   },
   "scripts": {
     "build": "tsc -b",
     "build:tests": "tsc -b test/tsconfig.json",
-    "clean": "rm -rf ./lib && tsc -b --clean",
+    "clean": "rm -rf ./dist && tsc -b --clean",
     "test": "vitest"
   }
 }

package/src/migrations/scripts/1771232439485-storage-version.ts

@@ -0,0 +1,44 @@
+import { migrations, storage } from '@powersync/service-core';
+import { openMigrationDB } from '../migration-utils.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+  await client.transaction(async (db) => {
+    await db.sql`
+      ALTER TABLE sync_rules
+      ADD COLUMN storage_version integer NOT NULL DEFAULT 1
+    `.execute();
+  });
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+  await client.transaction(async (db) => {
+    const newRules = await db.sql`
+      SELECT
+        id,
+        storage_version
+      FROM
+        sync_rules
+      WHERE
+        storage_version > ${{ type: 'int4', value: storage.LEGACY_STORAGE_VERSION }}
+    `.rows<{ id: number | bigint; storage_version: number | bigint }>();
+
+    if (newRules.length > 0) {
+      throw new Error(
+        `Cannot revert migration due to newer storage versions in use: ${newRules.map((r) => `${r.id}: v${r.storage_version}`).join(', ')}`
+      );
+    }
+
+    await db.sql`
+      ALTER TABLE sync_rules
+      DROP COLUMN storage_version
+    `.execute();
+  });
+};

package/src/migrations/scripts/1771424826685-current-data-pending-deletes.ts

@@ -0,0 +1,10 @@
+import { migrations } from '@powersync/service-core';
+
+export const up: migrations.PowerSyncMigrationFunction = async (_context) => {
+  // No-op.
+  // Pending-delete support is now storage-version specific and initialized when v3 sync rules are deployed.
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (_context) => {
+  // No-op.
+};

package/src/migrations/scripts/1771491856000-sync-plan.ts

@@ -0,0 +1,21 @@
+import { migrations } from '@powersync/service-core';
+import { openMigrationDB } from '../migration-utils.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+  await client.sql`
+    ALTER TABLE sync_rules
+    ADD COLUMN sync_plan JSON;
+  `.execute();
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+  await client.sql`ALTER TABLE sync_rules DROP COLUMN sync_plan`.execute();
+};

package/src/storage/PostgresBucketStorageFactory.ts

@@ -1,28 +1,23 @@
-import * as framework from '@powersync/lib-services-framework';
-import { GetIntanceOptions, storage, SyncRulesBucketStorage, UpdateSyncRulesOptions } from '@powersync/service-core';
+import { framework, GetIntanceOptions, storage, SyncRulesBucketStorage } from '@powersync/service-core';
 import * as pg_wire from '@powersync/service-jpgwire';
-import * as sync_rules from '@powersync/service-sync-rules';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
 
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import { models, NormalizedPostgresStorageConfig } from '../types/types.js';
 
+import { getStorageApplicationName } from '../utils/application-name.js';
 import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js';
 import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js';
 import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js';
 import { PostgresPersistedSyncRulesContent } from './sync-rules/PostgresPersistedSyncRulesContent.js';
-import { getStorageApplicationName } from '../utils/application-name.js';
 
 export type PostgresBucketStorageOptions = {
   config: NormalizedPostgresStorageConfig;
   slot_name_prefix: string;
 };
 
-export class PostgresBucketStorageFactory
-  extends framework.BaseObserver<storage.BucketStorageFactoryListener>
-  implements storage.BucketStorageFactory
-{
+export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
   readonly db: lib_postgres.DatabaseClient;
   public readonly slot_name_prefix: string;
 
@@ -87,15 +82,27 @@ export class PostgresBucketStorageFactory
 
     const sizes = await this.db.sql`
       SELECT
-        pg_total_relation_size('current_data') AS current_size_bytes,
+        COALESCE(
+          pg_total_relation_size(to_regclass('current_data')),
+          0
+        ) AS v1_current_size_bytes,
+        COALESCE(
+          pg_total_relation_size(to_regclass('v3_current_data')),
+          0
+        ) AS v3_current_size_bytes,
         pg_total_relation_size('bucket_parameters') AS parameter_size_bytes,
         pg_total_relation_size('bucket_data') AS operations_size_bytes;
-    `.first<{ current_size_bytes: bigint; parameter_size_bytes: bigint; operations_size_bytes: bigint }>();
+    `.first<{
+      v1_current_size_bytes: bigint;
+      v3_current_size_bytes: bigint;
+      parameter_size_bytes: bigint;
+      operations_size_bytes: bigint;
+    }>();
 
     return {
       operations_size_bytes: Number(sizes!.operations_size_bytes),
       parameters_size_bytes: Number(sizes!.parameter_size_bytes),
-      replication_size_bytes: Number(sizes!.current_size_bytes)
+      replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes)
     };
   }
 
@@ -145,42 +152,16 @@ export class PostgresBucketStorageFactory
     };
   }
 
-  // TODO possibly share implementation in abstract class
-  async configureSyncRules(options: UpdateSyncRulesOptions): Promise<{
-    updated: boolean;
-    persisted_sync_rules?: storage.PersistedSyncRulesContent;
-    lock?: storage.ReplicationLock;
-  }> {
-    const next = await this.getNextSyncRulesContent();
-    const active = await this.getActiveSyncRulesContent();
-
-    if (next?.sync_rules_content == options.content) {
-      framework.logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else if (next == null && active?.sync_rules_content == options.content) {
-      framework.logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else {
-      framework.logger.info('Sync rules updated from configuration');
-      const persisted_sync_rules = await this.updateSyncRules(options);
-      return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
-    }
-  }
-
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedSyncRulesContent> {
-    // TODO some shared implementation for this might be nice
-    if (options.validate) {
-      // Parse and validate before applying any changes
-      sync_rules.SqlSyncRules.fromYaml(options.content, {
-        // No schema-based validation at this point
-        schema: undefined,
-        defaultSchema: 'not_applicable', // Not needed for validation
-        throwOnError: true
-      });
-    } else {
-      // Apply unconditionally. Any errors will be reported via the diagnostics API.
+    const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+    const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
+    if (storageConfig == null) {
+      throw new framework.ServiceError(
+        framework.ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${storageVersion}`
+      );
     }
-
+    await this.initializeStorageVersion(storageConfig);
     return this.db.transaction(async (db) => {
       await db.sql`
         UPDATE sync_rules
@@ -197,7 +178,14 @@ export class PostgresBucketStorageFactory
               nextval('sync_rules_id_sequence') AS id
           )
         INSERT INTO
-          sync_rules (id, content, state, slot_name)
+          sync_rules (
+            id,
+            content,
+            sync_plan,
+            state,
+            slot_name,
+            storage_version
+          )
         VALUES
           (
             (
@@ -206,7 +194,8 @@ export class PostgresBucketStorageFactory
               FROM
                 next_id
             ),
-            ${{ type: 'varchar', value: options.content }},
+            ${{ type: 'varchar', value: options.config.yaml }},
+            ${{ type: 'json', value: options.config.plan }},
             ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
             CONCAT(
               ${{ type: 'varchar', value: this.slot_name_prefix }},
@@ -218,7 +207,8 @@ export class PostgresBucketStorageFactory
               ),
               '_',
               ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
-            )
+            ),
+            ${{ type: 'int4', value: storageVersion }}
           )
         RETURNING
           *
@@ -232,6 +222,34 @@ export class PostgresBucketStorageFactory
     });
   }
 
+  /**
+   * Lazy-initializes storage-version-specific structures, if needed.
+   */
+  private async initializeStorageVersion(storageConfig: storage.StorageVersionConfig) {
+    if (!storageConfig.softDeleteCurrentData) {
+      return;
+    }
+
+    await this.db.sql`
+      CREATE TABLE IF NOT EXISTS v3_current_data (
+        group_id integer NOT NULL,
+        source_table TEXT NOT NULL,
+        source_key bytea NOT NULL,
+        CONSTRAINT unique_v3_current_data_id PRIMARY KEY (group_id, source_table, source_key),
+        buckets jsonb NOT NULL,
+        data bytea NOT NULL,
+        lookups bytea[] NOT NULL,
+        pending_delete BIGINT NULL
+      )
+    `.execute();
+
+    await this.db.sql`
+      CREATE INDEX IF NOT EXISTS v3_current_data_pending_deletes ON v3_current_data (group_id, pending_delete)
+      WHERE
+        pending_delete IS NOT NULL
+    `.execute();
+  }
+
   async restartReplication(sync_rules_group_id: number): Promise<void> {
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();
@@ -240,10 +258,8 @@ export class PostgresBucketStorageFactory
     // The current one will continue serving sync requests until the next one has finished processing.
     if (next != null && next.id == sync_rules_group_id) {
       // We need to redo the "next" sync rules
-      await this.updateSyncRules({
-        content: next.sync_rules_content,
-        validate: false
-      });
+
+      await this.updateSyncRules(next.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sql`
         UPDATE sync_rules
@@ -255,10 +271,7 @@ export class PostgresBucketStorageFactory
       `.execute();
     } else if (next == null && active?.id == sync_rules_group_id) {
       // Slot removed for "active" sync rules, while there is no "next" one.
-      await this.updateSyncRules({
-        content: active.sync_rules_content,
-        validate: false
-      });
+      await this.updateSyncRules(active.asUpdateOptions());
 
       // Pro-actively stop replicating, but still serve clients with existing data
       await this.db.sql`
@@ -284,12 +297,6 @@ export class PostgresBucketStorageFactory
     }
   }
 
-  // TODO possibly share via abstract class
-  async getActiveSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getActiveSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getActiveSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null> {
     const activeRow = await this.db.sql`
       SELECT
@@ -313,12 +320,6 @@ export class PostgresBucketStorageFactory
     return new PostgresPersistedSyncRulesContent(this.db, activeRow);
   }
 
-  // TODO possibly share via abstract class
-  async getNextSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getNextSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getNextSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null> {
     const nextRow = await this.db.sql`
       SELECT

package/src/storage/PostgresCompactor.ts

@@ -51,20 +51,20 @@ export class PostgresCompactor {
   private moveBatchLimit: number;
   private moveBatchQueryLimit: number;
   private clearBatchLimit: number;
-  private maxOpId: InternalOpId | undefined;
+  private maxOpId: InternalOpId;
   private buckets: string[] | undefined;
 
   constructor(
     private db: lib_postgres.DatabaseClient,
     private group_id: number,
-    options?: PostgresCompactOptions
+    options: PostgresCompactOptions
   ) {
-    this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
-    this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
-    this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
-    this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
-    this.maxOpId = options?.maxOpId;
-    this.buckets = options?.compactBuckets;
+    this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
+    this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
+    this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
+    this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
+    this.maxOpId = options.maxOpId ?? 0n;
+    this.buckets = options.compactBuckets;
   }
 
   /**
@@ -75,37 +75,54 @@ export class PostgresCompactor {
   async compact() {
     if (this.buckets) {
       for (let bucket of this.buckets) {
-        // We can make this more efficient later on by iterating
-        // through the buckets in a single query.
-        // That makes batching more tricky, so we leave for later.
-        await this.compactInternal(bucket);
+        await this.compactSingleBucket(bucket);
       }
     } else {
-      await this.compactInternal(undefined);
+      await this.compactAllBuckets();
     }
   }
 
-  async compactInternal(bucket: string | undefined) {
-    const idLimitBytes = this.idLimitBytes;
+  private async compactAllBuckets() {
+    const DISCOVERY_BATCH_SIZE = 200;
+    let lastBucket = '';
+
+    while (true) {
+      const bucketRows = (await this.db.sql`
+        SELECT DISTINCT
+          bucket_name
+        FROM
+          bucket_data
+        WHERE
+          group_id = ${{ type: 'int4', value: this.group_id }}
+          AND bucket_name > ${{ type: 'varchar', value: lastBucket }}
+        ORDER BY
+          bucket_name ASC
+        LIMIT
+          ${{ type: 'int4', value: DISCOVERY_BATCH_SIZE }}
+      `.rows()) as { bucket_name: string }[];
 
-    let currentState: CurrentBucketState | null = null;
-
-    let bucketLower: string | null = null;
-    let bucketUpper: string | null = null;
-    const MAX_CHAR = String.fromCodePoint(0xffff);
-
-    if (bucket == null) {
-      bucketLower = '';
-      bucketUpper = MAX_CHAR;
-    } else if (bucket?.includes('[')) {
-      // Exact bucket name
-      bucketLower = bucket;
-      bucketUpper = bucket;
-    } else if (bucket) {
-      // Bucket definition name
-      bucketLower = `${bucket}[`;
-      bucketUpper = `${bucket}[${MAX_CHAR}`;
+      if (bucketRows.length === 0) {
+        break;
+      }
+
+      for (const row of bucketRows) {
+        await this.compactSingleBucket(row.bucket_name);
+      }
+
+      lastBucket = bucketRows[bucketRows.length - 1].bucket_name;
     }
+  }
+
+  private async compactSingleBucket(bucket: string) {
+    const idLimitBytes = this.idLimitBytes;
+
+    let currentState: CurrentBucketState = {
+      bucket: bucket,
+      seen: new Map(),
+      trackingSize: 0,
+      lastNotPut: null,
+      opsSincePut: 0
+    };
 
     let upperOpIdLimit = BIGINT_MAX;
 
@@ -123,16 +140,9 @@ export class PostgresCompactor {
           bucket_data
         WHERE
           group_id = ${{ type: 'int4', value: this.group_id }}
-          AND bucket_name >= ${{ type: 'varchar', value: bucketLower }}
-          AND (
-            (
-              bucket_name = ${{ type: 'varchar', value: bucketUpper }}
-              AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
-            )
-            OR bucket_name < ${{ type: 'varchar', value: bucketUpper }} COLLATE "C" -- Use binary comparison
-          )
+          AND bucket_name = ${{ type: 'varchar', value: bucket }}
+          AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
         ORDER BY
-          bucket_name DESC,
           op_id DESC
         LIMIT
           ${{ type: 'int4', value: this.moveBatchQueryLimit }}
@@ -150,32 +160,8 @@ export class PostgresCompactor {
       // Set upperBound for the next batch
       const lastBatchItem = batch[batch.length - 1];
       upperOpIdLimit = lastBatchItem.op_id;
-      bucketUpper = lastBatchItem.bucket_name;
 
       for (const doc of batch) {
-        if (currentState == null || doc.bucket_name != currentState.bucket) {
-          if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
-            // Important to flush before clearBucket()
-            await this.flush();
-            logger.info(
-              `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
-            );
-
-            const bucket = currentState.bucket;
-            const clearOp = currentState.lastNotPut;
-            // Free memory before clearing bucket
-            currentState = null;
-            await this.clearBucket(bucket, clearOp);
-          }
-          currentState = {
-            bucket: doc.bucket_name,
-            seen: new Map(),
-            trackingSize: 0,
-            lastNotPut: null,
-            opsSincePut: 0
-          };
-        }
-
         if (this.maxOpId != null && doc.op_id > this.maxOpId) {
           continue;
         }
@@ -237,16 +223,12 @@ export class PostgresCompactor {
     }
 
     await this.flush();
-    currentState?.seen.clear();
-    if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) {
+    currentState.seen.clear();
+    if (currentState.lastNotPut != null && currentState.opsSincePut > 1) {
       logger.info(
         `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
       );
-      const bucket = currentState.bucket;
-      const clearOp = currentState.lastNotPut;
-      // Free memory before clearing bucket
-      currentState = null;
-      await this.clearBucket(bucket, clearOp);
+      await this.clearBucket(currentState.bucket, currentState.lastNotPut);
     }
   }
 
@@ -258,6 +240,15 @@ export class PostgresCompactor {
     }
   }
 
+  /**
+   * Expose the internal clearBucket() method to tests.
+   *
+   * @deprecated Only for tests
+   */
+  clearBucketForTests(bucket: string, op: InternalOpId) {
+    return this.clearBucket(bucket, op);
+  }
+
   /**
    * Perform a CLEAR compact for a bucket.
    *