@powersync/service-module-postgres-storage 0.13.0 → 0.13.1

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -104,7 +104,7 @@ exports[`sync - postgres > storage v1 > compacting data - invalidate checkpoint
 ]
 `;
 
-exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for streams 1`] = `
+exports[`sync - postgres > storage v1 > encodes sync rules id in buckets for streams 1`] = `
 [
   {
     "checkpoint": {
@@ -159,13 +159,13 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre
 ]
 `;
 
-exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for streams 2`] = `
+exports[`sync - postgres > storage v1 > encodes sync rules id in buckets for streams 2`] = `
 [
   {
     "checkpoint": {
       "buckets": [
         {
-          "bucket": "2#test|0[]",
+          "bucket": "2#test2|0[]",
           "checksum": 920318466,
           "count": 1,
           "priority": 3,
@@ -181,7 +181,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre
         {
           "errors": [],
           "is_default": true,
-          "name": "test",
+          "name": "test2",
         },
       ],
       "write_checkpoint": undefined,
@@ -190,7 +190,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre
   {
     "data": {
       "after": "0",
-      "bucket": "2#test|0[]",
+      "bucket": "2#test2|0[]",
       "data": [
         {
           "checksum": 920318466,
@@ -199,7 +199,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre
           "object_type": "test",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+          "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441",
         },
       ],
       "has_more": false,
@@ -819,7 +819,7 @@ exports[`sync - postgres > storage v1 > sync updates to data query only 2`] = `
           "object_type": "lists",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+          "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f",
         },
       ],
       "has_more": false,
@@ -1026,7 +1026,7 @@ exports[`sync - postgres > storage v1 > sync updates to parameter query + data 2
           "object_type": "lists",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+          "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f",
         },
       ],
       "has_more": false,
@@ -1199,7 +1199,7 @@ exports[`sync - postgres > storage v2 > compacting data - invalidate checkpoint
 ]
 `;
 
-exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for streams 1`] = `
+exports[`sync - postgres > storage v2 > encodes sync rules id in buckets for streams 1`] = `
 [
   {
     "checkpoint": {
@@ -1254,13 +1254,13 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre
 ]
 `;
 
-exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for streams 2`] = `
+exports[`sync - postgres > storage v2 > encodes sync rules id in buckets for streams 2`] = `
 [
   {
     "checkpoint": {
       "buckets": [
         {
-          "bucket": "2#test|0[]",
+          "bucket": "2#test2|0[]",
           "checksum": 920318466,
           "count": 1,
           "priority": 3,
@@ -1276,7 +1276,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre
         {
           "errors": [],
           "is_default": true,
-          "name": "test",
+          "name": "test2",
         },
       ],
       "write_checkpoint": undefined,
@@ -1285,7 +1285,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre
   {
     "data": {
       "after": "0",
-      "bucket": "2#test|0[]",
+      "bucket": "2#test2|0[]",
       "data": [
         {
           "checksum": 920318466,
@@ -1294,7 +1294,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre
           "object_type": "test",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+          "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441",
         },
       ],
       "has_more": false,
@@ -1914,7 +1914,7 @@ exports[`sync - postgres > storage v2 > sync updates to data query only 2`] = `
           "object_type": "lists",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+          "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f",
         },
       ],
       "has_more": false,
@@ -2121,7 +2121,7 @@ exports[`sync - postgres > storage v2 > sync updates to parameter query + data 2
           "object_type": "lists",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+          "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f",
         },
       ],
       "has_more": false,
@@ -2294,7 +2294,7 @@ exports[`sync - postgres > storage v3 > compacting data - invalidate checkpoint
 ]
 `;
 
-exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 1`] = `
+exports[`sync - postgres > storage v3 > encodes sync rules id in buckets for streams 1`] = `
 [
   {
     "checkpoint": {
@@ -2349,13 +2349,13 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre
 ]
 `;
 
-exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 2`] = `
+exports[`sync - postgres > storage v3 > encodes sync rules id in buckets for streams 2`] = `
 [
   {
     "checkpoint": {
       "buckets": [
         {
-          "bucket": "2#test|0[]",
+          "bucket": "2#test2|0[]",
           "checksum": 920318466,
           "count": 1,
           "priority": 3,
@@ -2371,7 +2371,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre
         {
           "errors": [],
           "is_default": true,
-          "name": "test",
+          "name": "test2",
         },
       ],
       "write_checkpoint": undefined,
@@ -2380,7 +2380,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre
   {
     "data": {
       "after": "0",
-      "bucket": "2#test|0[]",
+      "bucket": "2#test2|0[]",
       "data": [
         {
           "checksum": 920318466,
@@ -2389,7 +2389,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre
           "object_type": "test",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+          "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441",
         },
       ],
       "has_more": false,
@@ -3009,7 +3009,7 @@ exports[`sync - postgres > storage v3 > sync updates to data query only 2`] = `
           "object_type": "lists",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+          "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f",
         },
       ],
       "has_more": false,
@@ -3216,7 +3216,7 @@ exports[`sync - postgres > storage v3 > sync updates to parameter query + data 2
           "object_type": "lists",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+          "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f",
         },
       ],
       "has_more": false,

package/test/src/env.ts

@@ -1,6 +1,6 @@
 import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
-  PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'),
+  PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'),
   CI: utils.type.boolean.default('false')
 });

package/test/src/storage.test.ts

@@ -1,5 +1,5 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequestMap, register, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
@@ -39,13 +39,15 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         )
       );
       const bucketStorage = factory.getInstance(syncRules);
+      const globalBucket = bucketRequest(syncRules, 'global[]');
 
-      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        const sourceTable = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY);
+      const result = await (async () => {
+        await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+        const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
 
         const largeDescription = '0123456789'.repeat(2_000_00);
 
-        await batch.save({
+        await writer.save({
           sourceTable,
           tag: storage.SaveOperationTag.INSERT,
           after: {
@@ -55,7 +57,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
           afterReplicaId: test_utils.rid('test1')
         });
 
-        await batch.save({
+        await writer.save({
           sourceTable,
           tag: storage.SaveOperationTag.INSERT,
           after: {
@@ -66,7 +68,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         });
 
         // Large enough to split the returned batch
-        await batch.save({
+        await writer.save({
           sourceTable,
           tag: storage.SaveOperationTag.INSERT,
           after: {
@@ -76,7 +78,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
           afterReplicaId: test_utils.rid('large2')
         });
 
-        await batch.save({
+        await writer.save({
           sourceTable,
           tag: storage.SaveOperationTag.INSERT,
           after: {
@@ -85,15 +87,14 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
           },
           afterReplicaId: test_utils.rid('test3')
         });
-      });
+        return writer.flush();
+      })();
 
       const checkpoint = result!.flushed_op;
 
       const options: storage.BucketDataBatchOptions = {};
 
-      const batch1 = await test_utils.fromAsync(
-        bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
-      );
+      const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options));
       expect(test_utils.getBatchData(batch1)).toEqual([
         { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
       ]);
@@ -106,7 +107,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
       const batch2 = await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
-          bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
+          [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
           options
         )
       );
@@ -122,7 +123,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
       const batch3 = await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
-          bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
+          [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
           options
         )
       );
@@ -138,7 +139,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
       const batch4 = await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
-          bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]),
+          [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],
           options
         )
       );

package/test/src/storage_compacting.test.ts

@@ -1,5 +1,5 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, bucketRequestMap, register, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
@@ -7,7 +7,6 @@ import { POSTGRES_STORAGE_FACTORY } from './util.js';
 describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY));
 
 describe('Postgres Compact - explicit bucket name', () => {
-  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY);
   test('compacts a specific bucket by exact name', async () => {
     await using factory = await POSTGRES_STORAGE_FACTORY.factory();
     const syncRules = await factory.updateSyncRules(
@@ -19,22 +18,26 @@ bucket_definitions:
     );
     const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      await batch.save({
-        sourceTable: TEST_TABLE,
+    const result = await (async () => {
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
+      await writer.save({
+        sourceTable: testTable,
         tag: storage.SaveOperationTag.INSERT,
         after: { id: 't1' },
         afterReplicaId: test_utils.rid('t1')
       });
-      await batch.save({
-        sourceTable: TEST_TABLE,
+      await writer.save({
+        sourceTable: testTable,
         tag: storage.SaveOperationTag.UPDATE,
         after: { id: 't1' },
         afterReplicaId: test_utils.rid('t1')
       });
-      await batch.markAllSnapshotDone('1/1');
-      await batch.commit('1/1');
-    });
+      await writer.markAllSnapshotDone('1/1');
+      const flushed = await writer.flush();
+      await writer.commit('1/1');
+      return flushed;
+    })();
 
     const checkpoint = result!.flushed_op;
 
@@ -46,7 +49,7 @@ bucket_definitions:
     });
 
     const batch = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
     );
 
     expect(batch.chunkData.data).toMatchObject([
@@ -69,34 +72,38 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
     const request = bucketRequest(syncRules, 'global[]');
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      await batch.markAllSnapshotDone('1/1');
-      await batch.save({
-        sourceTable: TEST_TABLE,
+    const result = await (async () => {
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
+      await writer.markAllSnapshotDone('1/1');
+      await writer.save({
+        sourceTable: testTable,
         tag: storage.SaveOperationTag.INSERT,
         after: { id: 't1' },
         afterReplicaId: test_utils.rid('t1')
       });
-      await batch.save({
-        sourceTable: TEST_TABLE,
+      await writer.save({
+        sourceTable: testTable,
         tag: storage.SaveOperationTag.DELETE,
         before: { id: 't1' },
         beforeReplicaId: test_utils.rid('t1')
       });
-      await batch.save({
-        sourceTable: TEST_TABLE,
+      await writer.save({
+        sourceTable: testTable,
         tag: storage.SaveOperationTag.INSERT,
         after: { id: 't2' },
         afterReplicaId: test_utils.rid('t2')
       });
-      await batch.save({
-        sourceTable: TEST_TABLE,
+      await writer.save({
+        sourceTable: testTable,
         tag: storage.SaveOperationTag.DELETE,
         before: { id: 't2' },
         beforeReplicaId: test_utils.rid('t2')
       });
-      await batch.commit('1/1');
-    });
+      const flushed = await writer.flush();
+      await writer.commit('1/1');
+      return flushed;
+    })();
 
     const checkpoint = result!.flushed_op;
     const rowsBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));

package/test/src/storage_sync.test.ts

@@ -1,5 +1,5 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, register, resolveTestTable, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
@@ -11,7 +11,6 @@ import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 function registerStorageVersionTests(storageVersion: number) {
   describe(`storage v${storageVersion}`, () => {
     const storageFactory = POSTGRES_STORAGE_FACTORY;
-    const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageFactory);
 
     register.registerSyncTests(storageFactory.factory, {
       storageVersion,
@@ -37,53 +36,55 @@ function registerStorageVersionTests(storageVersion: number) {
       const bucketStorage = factory.getInstance(syncRules);
       const globalBucket = bucketRequest(syncRules, 'global[]');
 
-      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        const sourceTable = TEST_TABLE;
-
-        const largeDescription = '0123456789'.repeat(2_000_00);
-
-        await batch.save({
-          sourceTable,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 'test1',
-            description: 'test1'
-          },
-          afterReplicaId: test_utils.rid('test1')
-        });
-
-        await batch.save({
-          sourceTable,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 'large1',
-            description: largeDescription
-          },
-          afterReplicaId: test_utils.rid('large1')
-        });
-
-        // Large enough to split the returned batch
-        await batch.save({
-          sourceTable,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 'large2',
-            description: largeDescription
-          },
-          afterReplicaId: test_utils.rid('large2')
-        });
-
-        await batch.save({
-          sourceTable,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 'test3',
-            description: 'test3'
-          },
-          afterReplicaId: test_utils.rid('test3')
-        });
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+
+      const sourceTable = await resolveTestTable(writer, 'test', ['id'], storageFactory);
+
+      const largeDescription = '0123456789'.repeat(2_000_00);
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test1',
+          description: 'test1'
+        },
+        afterReplicaId: test_utils.rid('test1')
+      });
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large1',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large1')
       });
 
+      // Large enough to split the returned batch
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large2',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large2')
+      });
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test3',
+          description: 'test3'
+        },
+        afterReplicaId: test_utils.rid('test3')
+      });
+
+      const result = await writer.flush();
+
       const checkpoint = result!.flushed_op;
 
       const options: storage.BucketDataBatchOptions = {};