@powersync/service-module-postgres-storage 0.11.2 → 0.12.0

package/test/src/migrations.test.ts

@@ -2,9 +2,10 @@ import { beforeEach, describe, expect, it } from 'vitest';
 
 import { Direction } from '@powersync/lib-services-framework';
 import { register } from '@powersync/service-core-tests';
+import { dropTables, PostgresBucketStorageFactory } from '../../src/index.js';
 import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js';
 import { env } from './env.js';
-import { POSTGRES_STORAGE_FACTORY, POSTGRES_STORAGE_SETUP } from './util.js';
+import { POSTGRES_STORAGE_FACTORY, POSTGRES_STORAGE_SETUP, TEST_CONNECTION_OPTIONS } from './util.js';
 
 const MIGRATION_AGENT_FACTORY = () => {
   return new PostgresMigrationAgent({ type: 'postgresql', uri: env.PG_STORAGE_TEST_URL, sslmode: 'disable' });
@@ -15,6 +16,12 @@ describe('Migrations', () => {
     // The migration tests clear the migration store, without running the down migrations.
     // This ensures all the down migrations have been run before.
     const setup = POSTGRES_STORAGE_SETUP;
+    await using factory = new PostgresBucketStorageFactory({
+      config: TEST_CONNECTION_OPTIONS,
+      slot_name_prefix: 'test_'
+    });
+
+    await dropTables(factory.db);
     await setup.migrate(Direction.Down);
   });
 

package/test/src/storage.test.ts

@@ -1,5 +1,5 @@
-import { storage } from '@powersync/service-core';
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequestMap, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
 
@@ -24,16 +24,16 @@ describe('Postgres Sync Bucket Storage - pg-specific', () => {
     // Test syncing a batch of data that is small in count,
     // but large enough in size to be split over multiple returned chunks.
     // Similar to the above test, but splits over 1MB chunks.
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await POSTGRES_STORAGE_FACTORY();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
     bucket_definitions:
       global:
         data:
           - SELECT id, description FROM "%"
-    `
+    `)
     );
-    await using factory = await POSTGRES_STORAGE_FACTORY();
-    const bucketStorage = factory.getInstance(sync_rules);
+    const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
@@ -87,7 +87,7 @@ describe('Postgres Sync Bucket Storage - pg-specific', () => {
     const options: storage.BucketDataBatchOptions = {};
 
     const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
     );
     expect(test_utils.getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -101,7 +101,7 @@ describe('Postgres Sync Bucket Storage - pg-specific', () => {
     const batch2 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([['global[]', BigInt(batch1[0].chunkData.next_after)]]),
+        bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
         options
       )
     );
@@ -117,7 +117,7 @@ describe('Postgres Sync Bucket Storage - pg-specific', () => {
     const batch3 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([['global[]', BigInt(batch2[0].chunkData.next_after)]]),
+        bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
         options
       )
     );
@@ -133,7 +133,7 @@ describe('Postgres Sync Bucket Storage - pg-specific', () => {
     const batch4 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([['global[]', BigInt(batch3[0].chunkData.next_after)]]),
+        bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]),
         options
       )
     );

package/test/src/storage_compacting.test.ts

@@ -1,5 +1,54 @@
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, bucketRequestMap, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
 
 describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY));
+
+describe('Postgres Compact - explicit bucket name', () => {
+  test('compacts a specific bucket by exact name', async () => {
+    await using factory = await POSTGRES_STORAGE_FACTORY();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+      `)
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.UPDATE,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.commit('1/1');
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    // Compact with an explicit bucket name — exercises the this.buckets
+    // iteration path, NOT the compactAllBuckets discovery path.
+    await bucketStorage.compact({
+      compactBuckets: [bucketRequest(syncRules, 'global[]')],
+      minBucketChanges: 1
+    });
+
+    const batch = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
+
+    expect(batch.chunkData.data).toMatchObject([
+      { op_id: '1', op: 'MOVE' },
+      { op_id: '2', op: 'PUT', object_id: 't1' }
+    ]);
+  });
+});

package/test/src/storage_sync.test.ts

@@ -1,12 +1,154 @@
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
-import { POSTGRES_STORAGE_FACTORY } from './util.js';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
+import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 /**
  * Bucket compacting is not yet implemented.
  * This causes the internal compacting test to fail.
  * Other tests have been verified manually.
  */
+function registerStorageVersionTests(storageVersion: number) {
+  describe(`storage v${storageVersion}`, () => {
+    const storageFactory = POSTGRES_STORAGE_FACTORY;
+
+    register.registerSyncTests(storageFactory, { storageVersion });
+
+    test('large batch (2)', async () => {
+      // Test syncing a batch of data that is small in count,
+      // but large enough in size to be split over multiple returned chunks.
+      // Similar to the above test, but splits over 1MB chunks.
+      await using factory = await storageFactory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM "%"
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const globalBucket = bucketRequest(syncRules, 'global[]');
+
+      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        const sourceTable = TEST_TABLE;
+
+        const largeDescription = '0123456789'.repeat(2_000_00);
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test1',
+            description: 'test1'
+          },
+          afterReplicaId: test_utils.rid('test1')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large1',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large1')
+        });
+
+        // Large enough to split the returned batch
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'large2',
+            description: largeDescription
+          },
+          afterReplicaId: test_utils.rid('large2')
+        });
+
+        await batch.save({
+          sourceTable,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 'test3',
+            description: 'test3'
+          },
+          afterReplicaId: test_utils.rid('test3')
+        });
+      });
+
+      const checkpoint = result!.flushed_op;
+
+      const options: storage.BucketDataBatchOptions = {};
+
+      const batch1 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(checkpoint, new Map([[globalBucket, 0n]]), options)
+      );
+      expect(test_utils.getBatchData(batch1)).toEqual([
+        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+      ]);
+      expect(test_utils.getBatchMeta(batch1)).toEqual({
+        after: '0',
+        has_more: true,
+        next_after: '1'
+      });
+
+      const batch2 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          new Map([[globalBucket, BigInt(batch1[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch2)).toEqual([
+        { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+      ]);
+      expect(test_utils.getBatchMeta(batch2)).toEqual({
+        after: '1',
+        has_more: true,
+        next_after: '2'
+      });
+
+      const batch3 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          new Map([[globalBucket, BigInt(batch2[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch3)).toEqual([
+        { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
+      ]);
+      expect(test_utils.getBatchMeta(batch3)).toEqual({
+        after: '2',
+        has_more: true,
+        next_after: '3'
+      });
+
+      const batch4 = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          new Map([[globalBucket, BigInt(batch3[0].chunkData.next_after)]]),
+          options
+        )
+      );
+      expect(test_utils.getBatchData(batch4)).toEqual([
+        { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+      ]);
+      expect(test_utils.getBatchMeta(batch4)).toEqual({
+        after: '3',
+        has_more: false,
+        next_after: '4'
+      });
+    });
+  });
+}
+
 describe('sync - postgres', () => {
-  register.registerSyncTests(POSTGRES_STORAGE_FACTORY);
+  for (const storageVersion of TEST_STORAGE_VERSIONS) {
+    registerStorageVersionTests(storageVersion);
+  }
 });

package/test/src/util.ts

@@ -3,6 +3,7 @@ import { fileURLToPath } from 'url';
 import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js';
 import { env } from './env.js';
 import { postgresTestSetup } from '../../src/utils/test-utils.js';
+import { CURRENT_STORAGE_VERSION, LEGACY_STORAGE_VERSION } from '@powersync/service-core';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -34,3 +35,5 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({
 
 export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.factory;
 export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory;
+
+export const TEST_STORAGE_VERSIONS = [LEGACY_STORAGE_VERSION, CURRENT_STORAGE_VERSION];

package/test/src/__snapshots__/storage.test.ts.snap

@@ -1,9 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`Postgres Sync Bucket Storage - Data > empty storage metrics 1`] = `
-{
-  "operations_size_bytes": 16384,
-  "parameters_size_bytes": 32768,
-  "replication_size_bytes": 16384,
-}
-`;