@powersync/service-module-postgres-storage 0.12.0 → 0.13.0

package/test/src/migrations.test.ts

@@ -28,7 +28,7 @@ describe('Migrations', () => {
   register.registerMigrationTests(MIGRATION_AGENT_FACTORY);
 
   it('Should have tables declared', async () => {
-    const { db } = await POSTGRES_STORAGE_FACTORY();
+    const { db } = await POSTGRES_STORAGE_FACTORY.factory();
 
     const tables = await db.sql`
       SELECT

package/test/src/storage.test.ts

@@ -1,149 +1,155 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequestMap, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { bucketRequestMap, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
-import { POSTGRES_STORAGE_FACTORY } from './util.js';
-
-describe('Postgres Sync Bucket Storage - Parameters', () =>
-  register.registerDataStorageParameterTests(POSTGRES_STORAGE_FACTORY));
-
-describe('Postgres Sync Bucket Storage - Data', () => register.registerDataStorageDataTests(POSTGRES_STORAGE_FACTORY));
-
-describe('Postgres Sync Bucket Storage - Checkpoints', () =>
-  register.registerDataStorageCheckpointTests(POSTGRES_STORAGE_FACTORY));
+import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 
-describe('Postgres Sync Bucket Storage - pg-specific', () => {
-  /**
-   * The split of returned results can vary depending on storage drivers.
-   * The large rows here are 2MB large while the default chunk limit is 1mb.
-   * The Postgres storage driver will detect if the next row will increase the batch
-   * over the limit and separate that row into a new batch (or single row batch) if applicable.
-   */
-  test('large batch (2)', async () => {
-    // Test syncing a batch of data that is small in count,
-    // but large enough in size to be split over multiple returned chunks.
-    // Similar to the above test, but splits over 1MB chunks.
-    await using factory = await POSTGRES_STORAGE_FACTORY();
-    const syncRules = await factory.updateSyncRules(
-      updateSyncRulesFromYaml(`
+for (let storageVersion of TEST_STORAGE_VERSIONS) {
+  describe(`Postgres Sync Bucket Storage - Parameters - v${storageVersion}`, () =>
+    register.registerDataStorageParameterTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
+
+  describe(`Postgres Sync Bucket Storage - Data - v${storageVersion}`, () =>
+    register.registerDataStorageDataTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
+
+  describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
+    register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
+
+  describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => {
+    /**
+     * The split of returned results can vary depending on storage drivers.
+     * The large rows here are 2MB large while the default chunk limit is 1mb.
+     * The Postgres storage driver will detect if the next row will increase the batch
+     * over the limit and separate that row into a new batch (or single row batch) if applicable.
+     */
+    test('large batch (2)', async () => {
+      // Test syncing a batch of data that is small in count,
+      // but large enough in size to be split over multiple returned chunks.
+      // Similar to the above test, but splits over 1MB chunks.
+      await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
     bucket_definitions:
       global:
         data:
           - SELECT id, description FROM "%"
-    `)
-    );
-    const bucketStorage = factory.getInstance(syncRules);
-
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      const sourceTable = TEST_TABLE;
-
-      const largeDescription = '0123456789'.repeat(2_000_00);
-
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test1',
-          description: 'test1'
-        },
-        afterReplicaId: test_utils.rid('test1')
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+
+      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        const sourceTable = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY);
+
+        const largeDescription = '0123456789'.repeat(2_000_00);
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test1',
+            description: 'test1'
+          },
+          afterReplicaId: test_utils.rid('test1')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large1',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large1')
+        });
+
+        // Large enough to split the returned batch
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large2',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large2')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test3',
+            description: 'test3'
+          },
+          afterReplicaId: test_utils.rid('test3')
+        });
       });
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large1',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large1')
-      });
+      const checkpoint = result!.flushed_op;
 
-      // Large enough to split the returned batch
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large2',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large2')
-      });
+      const options: storage.BucketDataBatchOptions = {};
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test3',
-          description: 'test3'
-        },
-        afterReplicaId: test_utils.rid('test3')
+      const batch1 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
+      );
+      expect(test_utils.getBatchData(batch1)).toEqual([
+        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+      ]);
+      expect(test_utils.getBatchMeta(batch1)).toEqual({
+        after: '0',
+        has_more: true,
+        next_after: '1'
       });
-    });
-
-    const checkpoint = result!.flushed_op;
-
-    const options: storage.BucketDataBatchOptions = {};
 
-    const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
-    );
-    expect(test_utils.getBatchData(batch1)).toEqual([
-      { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
-    ]);
-    expect(test_utils.getBatchMeta(batch1)).toEqual({
-      after: '0',
-      has_more: true,
-      next_after: '1'
-    });
-
-    const batch2 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(
-        checkpoint,
-        bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
-        options
-      )
-    );
-    expect(test_utils.getBatchData(batch2)).toEqual([
-      { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
-    ]);
-    expect(test_utils.getBatchMeta(batch2)).toEqual({
-      after: '1',
-      has_more: true,
-      next_after: '2'
-    });
+      const batch2 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch2)).toEqual([
+        { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+      ]);
+      expect(test_utils.getBatchMeta(batch2)).toEqual({
+        after: '1',
+        has_more: true,
+        next_after: '2'
+      });
 
-    const batch3 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(
-        checkpoint,
-        bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
-        options
-      )
-    );
-    expect(test_utils.getBatchData(batch3)).toEqual([
-      { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
-    ]);
-    expect(test_utils.getBatchMeta(batch3)).toEqual({
-      after: '2',
-      has_more: true,
-      next_after: '3'
-    });
+      const batch3 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch3)).toEqual([
+        { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
+      ]);
+      expect(test_utils.getBatchMeta(batch3)).toEqual({
+        after: '2',
+        has_more: true,
+        next_after: '3'
+      });
 
-    const batch4 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(
-        checkpoint,
-        bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]),
-        options
-      )
-    );
-    expect(test_utils.getBatchData(batch4)).toEqual([
-      { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
-    ]);
-    expect(test_utils.getBatchMeta(batch4)).toEqual({
-      after: '3',
-      has_more: false,
-      next_after: '4'
+      const batch4 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch4)).toEqual([
+        { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+      ]);
+      expect(test_utils.getBatchMeta(batch4)).toEqual({
+        after: '3',
+        has_more: false,
+        next_after: '4'
+      });
     });
   });
-});
+}

package/test/src/storage_compacting.test.ts

@@ -1,13 +1,15 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, bucketRequestMap, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, bucketRequestMap, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
+import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
 
 describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY));
 
 describe('Postgres Compact - explicit bucket name', () => {
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY);
   test('compacts a specific bucket by exact name', async () => {
-    await using factory = await POSTGRES_STORAGE_FACTORY();
+    await using factory = await POSTGRES_STORAGE_FACTORY.factory();
     const syncRules = await factory.updateSyncRules(
       updateSyncRulesFromYaml(`
 bucket_definitions:
@@ -30,6 +32,7 @@ bucket_definitions:
         after: { id: 't1' },
         afterReplicaId: test_utils.rid('t1')
       });
+      await batch.markAllSnapshotDone('1/1');
       await batch.commit('1/1');
     });
 
@@ -38,7 +41,7 @@ bucket_definitions:
     // Compact with an explicit bucket name — exercises the this.buckets
     // iteration path, NOT the compactAllBuckets discovery path.
     await bucketStorage.compact({
-      compactBuckets: [bucketRequest(syncRules, 'global[]')],
+      compactBuckets: [bucketRequest(syncRules, 'global[]').bucket],
       minBucketChanges: 1
     });
 
@@ -51,4 +54,63 @@ bucket_definitions:
       { op_id: '2', op: 'PUT', object_id: 't1' }
     ]);
   });
+
+  test('clearBucket fails fast when prefix includes PUT', async () => {
+    // This tests the specific implementation, to check that our operation type guard is working
+    // for CLEAR compacting.
+    await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+      `)
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+    const request = bucketRequest(syncRules, 'global[]');
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.DELETE,
+        before: { id: 't1' },
+        beforeReplicaId: test_utils.rid('t1')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't2' },
+        afterReplicaId: test_utils.rid('t2')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.DELETE,
+        before: { id: 't2' },
+        beforeReplicaId: test_utils.rid('t2')
+      });
+      await batch.commit('1/1');
+    });
+
+    const checkpoint = result!.flushed_op;
+    const rowsBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const dataBefore = test_utils.getBatchData(rowsBefore);
+    const clearToOpId = BigInt(dataBefore[2].op_id);
+
+    const compactor = new PostgresCompactor(factory.db, bucketStorage.group_id, {});
+    // Trigger the private method directly
+    await expect(compactor.clearBucketForTests(request.bucket, clearToOpId)).rejects.toThrow(
+      /Unexpected PUT operation/
+    );
+
+    // The method wraps in a transaction; on assertion error the bucket must remain unchanged.
+    const rowsAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    expect(test_utils.getBatchData(rowsAfter)).toEqual(dataBefore);
+  });
 });

package/test/src/storage_sync.test.ts

@@ -1,5 +1,5 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
@@ -11,14 +11,18 @@ import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 function registerStorageVersionTests(storageVersion: number) {
   describe(`storage v${storageVersion}`, () => {
     const storageFactory = POSTGRES_STORAGE_FACTORY;
+    const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageFactory);
 
-    register.registerSyncTests(storageFactory, { storageVersion });
+    register.registerSyncTests(storageFactory.factory, {
+      storageVersion,
+      tableIdStrings: storageFactory.tableIdStrings
+    });
 
     test('large batch (2)', async () => {
       // Test syncing a batch of data that is small in count,
       // but large enough in size to be split over multiple returned chunks.
       // Similar to the above test, but splits over 1MB chunks.
-      await using factory = await storageFactory();
+      await using factory = await storageFactory.factory();
       const syncRules = await factory.updateSyncRules(
         updateSyncRulesFromYaml(
           `
@@ -84,9 +88,7 @@ function registerStorageVersionTests(storageVersion: number) {
 
       const options: storage.BucketDataBatchOptions = {};
 
-      const batch1 = await test_utils.fromAsync(
-        bucketStorage.getBucketDataBatch(checkpoint, new Map([[globalBucket, 0n]]), options)
-      );
+      const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options));
       expect(test_utils.getBatchData(batch1)).toEqual([
         { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
       ]);
@@ -99,7 +101,7 @@ function registerStorageVersionTests(storageVersion: number) {
       const batch2 = await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
-          new Map([[globalBucket, BigInt(batch1[0].chunkData.next_after)]]),
+          [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
           options
         )
       );
@@ -115,7 +117,7 @@ function registerStorageVersionTests(storageVersion: number) {
       const batch3 = await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
-          new Map([[globalBucket, BigInt(batch2[0].chunkData.next_after)]]),
+          [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
           options
         )
       );
@@ -131,7 +133,7 @@ function registerStorageVersionTests(storageVersion: number) {
       const batch4 = await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
-          new Map([[globalBucket, BigInt(batch3[0].chunkData.next_after)]]),
+          [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],
           options
         )
       );

package/test/src/util.ts

@@ -1,9 +1,9 @@
+import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js';
-import { env } from './env.js';
 import { postgresTestSetup } from '../../src/utils/test-utils.js';
-import { CURRENT_STORAGE_VERSION, LEGACY_STORAGE_VERSION } from '@powersync/service-core';
+import { env } from './env.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -33,7 +33,7 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({
   migrationAgent: (config) => new TestPostgresMigrationAgent(config)
 });
 
-export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.factory;
+export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP;
 export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory;
 
-export const TEST_STORAGE_VERSIONS = [LEGACY_STORAGE_VERSION, CURRENT_STORAGE_VERSION];
+export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;