@powersync/service-module-postgres-storage 0.11.2 → 0.13.0

package/test/src/migrations.test.ts

@@ -2,9 +2,10 @@ import { beforeEach, describe, expect, it } from 'vitest';
 
 import { Direction } from '@powersync/lib-services-framework';
 import { register } from '@powersync/service-core-tests';
+import { dropTables, PostgresBucketStorageFactory } from '../../src/index.js';
 import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js';
 import { env } from './env.js';
-import { POSTGRES_STORAGE_FACTORY, POSTGRES_STORAGE_SETUP } from './util.js';
+import { POSTGRES_STORAGE_FACTORY, POSTGRES_STORAGE_SETUP, TEST_CONNECTION_OPTIONS } from './util.js';
 
 const MIGRATION_AGENT_FACTORY = () => {
   return new PostgresMigrationAgent({ type: 'postgresql', uri: env.PG_STORAGE_TEST_URL, sslmode: 'disable' });
@@ -15,13 +16,19 @@ describe('Migrations', () => {
     // The migration tests clear the migration store, without running the down migrations.
     // This ensures all the down migrations have been run before.
     const setup = POSTGRES_STORAGE_SETUP;
+    await using factory = new PostgresBucketStorageFactory({
+      config: TEST_CONNECTION_OPTIONS,
+      slot_name_prefix: 'test_'
+    });
+
+    await dropTables(factory.db);
     await setup.migrate(Direction.Down);
   });
 
   register.registerMigrationTests(MIGRATION_AGENT_FACTORY);
 
   it('Should have tables declared', async () => {
-    const { db } = await POSTGRES_STORAGE_FACTORY();
+    const { db } = await POSTGRES_STORAGE_FACTORY.factory();
 
     const tables = await db.sql`
       SELECT

package/test/src/storage.test.ts

@@ -1,149 +1,155 @@
-import { storage } from '@powersync/service-core';
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequestMap, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
-import { POSTGRES_STORAGE_FACTORY } from './util.js';
-
-describe('Postgres Sync Bucket Storage - Parameters', () =>
-  register.registerDataStorageParameterTests(POSTGRES_STORAGE_FACTORY));
-
-describe('Postgres Sync Bucket Storage - Data', () => register.registerDataStorageDataTests(POSTGRES_STORAGE_FACTORY));
-
-describe('Postgres Sync Bucket Storage - Checkpoints', () =>
-  register.registerDataStorageCheckpointTests(POSTGRES_STORAGE_FACTORY));
+import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 
-describe('Postgres Sync Bucket Storage - pg-specific', () => {
-  /**
-   * The split of returned results can vary depending on storage drivers.
-   * The large rows here are 2MB large while the default chunk limit is 1mb.
-   * The Postgres storage driver will detect if the next row will increase the batch
-   * over the limit and separate that row into a new batch (or single row batch) if applicable.
-   */
-  test('large batch (2)', async () => {
-    // Test syncing a batch of data that is small in count,
-    // but large enough in size to be split over multiple returned chunks.
-    // Similar to the above test, but splits over 1MB chunks.
-    const sync_rules = test_utils.testRules(
-      `
+for (let storageVersion of TEST_STORAGE_VERSIONS) {
+  describe(`Postgres Sync Bucket Storage - Parameters - v${storageVersion}`, () =>
+    register.registerDataStorageParameterTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
+
+  describe(`Postgres Sync Bucket Storage - Data - v${storageVersion}`, () =>
+    register.registerDataStorageDataTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
+
+  describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
+    register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
+
+  describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => {
+    /**
+     * The split of returned results can vary depending on storage drivers.
+     * The large rows here are 2MB large while the default chunk limit is 1mb.
+     * The Postgres storage driver will detect if the next row will increase the batch
+     * over the limit and separate that row into a new batch (or single row batch) if applicable.
+     */
+    test('large batch (2)', async () => {
+      // Test syncing a batch of data that is small in count,
+      // but large enough in size to be split over multiple returned chunks.
+      // Similar to the above test, but splits over 1MB chunks.
+      await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
     bucket_definitions:
       global:
         data:
           - SELECT id, description FROM "%"
-    `
-    );
-    await using factory = await POSTGRES_STORAGE_FACTORY();
-    const bucketStorage = factory.getInstance(sync_rules);
-
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      const sourceTable = TEST_TABLE;
-
-      const largeDescription = '0123456789'.repeat(2_000_00);
-
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test1',
-          description: 'test1'
-        },
-        afterReplicaId: test_utils.rid('test1')
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+
+      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        const sourceTable = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY);
+
+        const largeDescription = '0123456789'.repeat(2_000_00);
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test1',
+            description: 'test1'
+          },
+          afterReplicaId: test_utils.rid('test1')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large1',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large1')
+        });
+
+        // Large enough to split the returned batch
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large2',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large2')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test3',
+            description: 'test3'
+          },
+          afterReplicaId: test_utils.rid('test3')
+        });
       });
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large1',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large1')
-      });
+      const checkpoint = result!.flushed_op;
 
-      // Large enough to split the returned batch
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large2',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large2')
-      });
+      const options: storage.BucketDataBatchOptions = {};
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test3',
-          description: 'test3'
-        },
-        afterReplicaId: test_utils.rid('test3')
+      const batch1 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
+      );
+      expect(test_utils.getBatchData(batch1)).toEqual([
+        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+      ]);
+      expect(test_utils.getBatchMeta(batch1)).toEqual({
+        after: '0',
+        has_more: true,
+        next_after: '1'
       });
-    });
-
-    const checkpoint = result!.flushed_op;
-
-    const options: storage.BucketDataBatchOptions = {};
 
-    const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), options)
-    );
-    expect(test_utils.getBatchData(batch1)).toEqual([
-      { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
-    ]);
-    expect(test_utils.getBatchMeta(batch1)).toEqual({
-      after: '0',
-      has_more: true,
-      next_after: '1'
-    });
-
-    const batch2 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(
-        checkpoint,
-        new Map([['global[]', BigInt(batch1[0].chunkData.next_after)]]),
-        options
-      )
-    );
-    expect(test_utils.getBatchData(batch2)).toEqual([
-      { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
-    ]);
-    expect(test_utils.getBatchMeta(batch2)).toEqual({
-      after: '1',
-      has_more: true,
-      next_after: '2'
-    });
+      const batch2 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch2)).toEqual([
+        { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+      ]);
+      expect(test_utils.getBatchMeta(batch2)).toEqual({
+        after: '1',
+        has_more: true,
+        next_after: '2'
+      });
 
-    const batch3 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(
-        checkpoint,
-        new Map([['global[]', BigInt(batch2[0].chunkData.next_after)]]),
-        options
-      )
-    );
-    expect(test_utils.getBatchData(batch3)).toEqual([
-      { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
-    ]);
-    expect(test_utils.getBatchMeta(batch3)).toEqual({
-      after: '2',
-      has_more: true,
-      next_after: '3'
-    });
+      const batch3 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch3)).toEqual([
+        { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
+      ]);
+      expect(test_utils.getBatchMeta(batch3)).toEqual({
+        after: '2',
+        has_more: true,
+        next_after: '3'
+      });
 
-    const batch4 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(
-        checkpoint,
-        new Map([['global[]', BigInt(batch3[0].chunkData.next_after)]]),
-        options
-      )
-    );
-    expect(test_utils.getBatchData(batch4)).toEqual([
-      { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
-    ]);
-    expect(test_utils.getBatchMeta(batch4)).toEqual({
-      after: '3',
-      has_more: false,
-      next_after: '4'
+      const batch4 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch4)).toEqual([
+        { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+      ]);
+      expect(test_utils.getBatchMeta(batch4)).toEqual({
+        after: '3',
+        has_more: false,
+        next_after: '4'
+      });
     });
   });
-});
+}

package/test/src/storage_compacting.test.ts

@@ -1,5 +1,116 @@
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, bucketRequestMap, register, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
+import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
 
 describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY));
+
+describe('Postgres Compact - explicit bucket name', () => {
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY);
+  test('compacts a specific bucket by exact name', async () => {
+    await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+      `)
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.UPDATE,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.markAllSnapshotDone('1/1');
+      await batch.commit('1/1');
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    // Compact with an explicit bucket name — exercises the this.buckets
+    // iteration path, NOT the compactAllBuckets discovery path.
+    await bucketStorage.compact({
+      compactBuckets: [bucketRequest(syncRules, 'global[]').bucket],
+      minBucketChanges: 1
+    });
+
+    const batch = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
+
+    expect(batch.chunkData.data).toMatchObject([
+      { op_id: '1', op: 'MOVE' },
+      { op_id: '2', op: 'PUT', object_id: 't1' }
+    ]);
+  });
+
+  test('clearBucket fails fast when prefix includes PUT', async () => {
+    // This tests the specific implementation, to check that our operation type guard is working
+    // for CLEAR compacting.
+    await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+      `)
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+    const request = bucketRequest(syncRules, 'global[]');
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.DELETE,
+        before: { id: 't1' },
+        beforeReplicaId: test_utils.rid('t1')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't2' },
+        afterReplicaId: test_utils.rid('t2')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.DELETE,
+        before: { id: 't2' },
+        beforeReplicaId: test_utils.rid('t2')
+      });
+      await batch.commit('1/1');
+    });
+
+    const checkpoint = result!.flushed_op;
+    const rowsBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const dataBefore = test_utils.getBatchData(rowsBefore);
+    const clearToOpId = BigInt(dataBefore[2].op_id);
+
+    const compactor = new PostgresCompactor(factory.db, bucketStorage.group_id, {});
+    // Trigger the private method directly
+    await expect(compactor.clearBucketForTests(request.bucket, clearToOpId)).rejects.toThrow(
+      /Unexpected PUT operation/
+    );
+
+    // The method wraps in a transaction; on assertion error the bucket must remain unchanged.
+    const rowsAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    expect(test_utils.getBatchData(rowsAfter)).toEqual(dataBefore);
+  });
+});

package/test/src/storage_sync.test.ts

@@ -1,12 +1,156 @@
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
-import { POSTGRES_STORAGE_FACTORY } from './util.js';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
+import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 /**
  * Bucket compacting is not yet implemented.
  * This causes the internal compacting test to fail.
  * Other tests have been verified manually.
  */
+function registerStorageVersionTests(storageVersion: number) {
+  describe(`storage v${storageVersion}`, () => {
+    const storageFactory = POSTGRES_STORAGE_FACTORY;
+    const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageFactory);
+
+    register.registerSyncTests(storageFactory.factory, {
+      storageVersion,
+      tableIdStrings: storageFactory.tableIdStrings
+    });
+
+    test('large batch (2)', async () => {
+      // Test syncing a batch of data that is small in count,
+      // but large enough in size to be split over multiple returned chunks.
+      // Similar to the above test, but splits over 1MB chunks.
+      await using factory = await storageFactory.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM "%"
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const globalBucket = bucketRequest(syncRules, 'global[]');
+
+      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        const sourceTable = TEST_TABLE;
+
+        const largeDescription = '0123456789'.repeat(2_000_00);
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test1',
+            description: 'test1'
+          },
+          afterReplicaId: test_utils.rid('test1')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large1',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large1')
+        });
+
+        // Large enough to split the returned batch
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large2',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large2')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test3',
+            description: 'test3'
+          },
+          afterReplicaId: test_utils.rid('test3')
+        });
+      });
+
+      const checkpoint = result!.flushed_op;
+
+      const options: storage.BucketDataBatchOptions = {};
+
+      const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options));
+      expect(test_utils.getBatchData(batch1)).toEqual([
+        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+      ]);
+      expect(test_utils.getBatchMeta(batch1)).toEqual({
+        after: '0',
+        has_more: true,
+        next_after: '1'
+      });
+
+      const batch2 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch2)).toEqual([
+        { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+      ]);
+      expect(test_utils.getBatchMeta(batch2)).toEqual({
+        after: '1',
+        has_more: true,
+        next_after: '2'
+      });
+
+      const batch3 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch3)).toEqual([
+        { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
+      ]);
+      expect(test_utils.getBatchMeta(batch3)).toEqual({
+        after: '2',
+        has_more: true,
+        next_after: '3'
+      });
+
+      const batch4 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch4)).toEqual([
+        { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+      ]);
+      expect(test_utils.getBatchMeta(batch4)).toEqual({
+        after: '3',
+        has_more: false,
+        next_after: '4'
+      });
+    });
+  });
+}
+
 describe('sync - postgres', () => {
-  register.registerSyncTests(POSTGRES_STORAGE_FACTORY);
+  for (const storageVersion of TEST_STORAGE_VERSIONS) {
+    registerStorageVersionTests(storageVersion);
+  }
 });

package/test/src/util.ts

@@ -1,8 +1,9 @@
+import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js';
-import { env } from './env.js';
 import { postgresTestSetup } from '../../src/utils/test-utils.js';
+import { env } from './env.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -32,5 +33,7 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({
   migrationAgent: (config) => new TestPostgresMigrationAgent(config)
 });
 
-export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.factory;
+export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP;
 export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory;
+
+export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;