@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/test/src/storage.test.ts

@@ -1,8 +1,15 @@
-import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
+import { framework, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests';
+import { orNull } from '@powersync/service-types';
+import * as t from 'ts-codec';
 import { describe, expect, test } from 'vitest';
+import { CLEAR_BATCH_LIMIT } from '../../src/storage/PostgresSyncRulesStorage.js';
 import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
+const CheckpointRequestedAtRow = t.object({
+  checkpoint_requested_at: orNull(framework.codecs.date)
+});
+
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 
 for (let storageVersion of TEST_STORAGE_VERSIONS) {
@@ -10,12 +17,226 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
     register.registerDataStorageParameterTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
 
   describe(`Postgres Sync Bucket Storage - Data - v${storageVersion}`, () =>
-    register.registerDataStorageDataTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
+    register.registerDataStorageDataTests({
+      ...POSTGRES_STORAGE_FACTORY,
+      storageVersion,
+      compressedBucketStorage: false
+    }));
 
   describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
     register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
 
   describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => {
+    test('uses checkpoint_requested_at as the client-requested checkpoint marker', async () => {
+      await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data: []
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const requestedAt = async (userId = 'user1') =>
+        (
+          await factory.db.sql`
+            SELECT
+              checkpoint_requested_at
+            FROM
+              write_checkpoints
+            WHERE
+              user_id = ${{ type: 'varchar', value: userId }}
+          `
+            .decoded(CheckpointRequestedAtRow)
+            .first()
+        )?.checkpoint_requested_at;
+
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '5/0' }, checkpoint_request_id: 42n }
+      ]);
+      const requested = await requestedAt();
+      expect(requested).toBeInstanceOf(Date);
+
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 41n }
+      ]);
+      await expect(requestedAt()).resolves.toEqual(requested);
+
+      const expiredRequestedAt = new Date('2024-01-01T00:00:00.000Z');
+      await factory.db.sql`
+        UPDATE write_checkpoints
+        SET
+          checkpoint_requested_at = ${{ type: 1184, value: expiredRequestedAt.toISOString() }}
+        WHERE
+          user_id = 'user1'
+      `.execute();
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 42n }
+      ]);
+      // Retrying the current id refreshes its retention timestamp. The shared
+      // checkpoint tests verify that the original source head is preserved.
+      expect((await requestedAt())!.getTime()).toBeGreaterThan(expiredRequestedAt.getTime());
+
+      await factory.db.sql`
+        UPDATE write_checkpoints
+        SET
+          checkpoint_requested_at = ${{ type: 1184, value: expiredRequestedAt.toISOString() }}
+        WHERE
+          user_id = 'user1'
+      `.execute();
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 43n }
+      ]);
+      // A greater id refreshes retention while advancing the stored checkpoint.
+      expect((await requestedAt())!.getTime()).toBeGreaterThan(expiredRequestedAt.getTime());
+
+      await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '7/0' } }]);
+      await expect(requestedAt()).resolves.toBeNull();
+
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user2', heads: { '1': '8/0' }, checkpoint_request_id: 50n }
+      ]);
+      await factory.db.sql`
+        UPDATE write_checkpoints
+        SET
+          checkpoint_requested_at = ${{ type: 1184, value: '2024-01-01T00:00:00.000Z' }}
+        WHERE
+          user_id = 'user2'
+      `.execute();
+      {
+        await using activationWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+        await activationWriter.markAllSnapshotDone('1/1');
+        await activationWriter.keepalive('1/1');
+      }
+      await compactActive(factory, {
+        compactBuckets: [],
+        deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z')
+      });
+      // Compaction removes expired requests based on the refreshed timestamp.
+      await expect(requestedAt('user2')).resolves.toBeUndefined();
+
+      const customRequestedAt = async (userId = 'custom1') =>
+        (
+          await factory.db.sql`
+            SELECT
+              checkpoint_requested_at
+            FROM
+              custom_write_checkpoints
+            WHERE
+              user_id = ${{ type: 'varchar', value: userId }}
+          `
+            .decoded(CheckpointRequestedAtRow)
+            .first()
+        )?.checkpoint_requested_at;
+
+      bucketStorage.setWriteCheckpointMode({
+        mode: storage.WriteCheckpointMode.CUSTOM
+      });
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      await writer.markAllSnapshotDone('1/1');
+      const customCheckpointRequestedAt = new Date('2024-01-01T00:00:00.000Z');
+      writer.addCustomWriteCheckpoint({
+        user_id: 'custom1',
+        checkpoint: 51n,
+        checkpoint_requested_at: customCheckpointRequestedAt
+      });
+      await writer.flush();
+      await expect(customRequestedAt()).resolves.toEqual(customCheckpointRequestedAt);
+
+      writer.addCustomWriteCheckpoint({
+        user_id: 'custom1',
+        checkpoint: 52n
+      });
+      await writer.flush();
+      await expect(customRequestedAt()).resolves.toBeNull();
+    });
+
+    test('clears storage in batches', async () => {
+      await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data: []
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const groupId = bucketStorage.replicationStreamId;
+      const otherGroupId = groupId + 1;
+      const currentDataTable = bucketStorage.storageConfig.softDeleteCurrentData ? 'v3_current_data' : 'current_data';
+
+      // Seed more than one batch of bucket data, plus rows in the other tables,
+      // directly - clear() only depends on group_id.
+      const seed = async (gid: number, bucketDataRows: number) => {
+        await factory.db.query({
+          statement: `
+            INSERT INTO bucket_data (group_id, bucket_name, op_id, op, checksum)
+            SELECT $1, 'global[]', i, 'PUT', 0 FROM generate_series(1, $2) i
+          `,
+          params: [
+            { type: 'int4', value: gid },
+            { type: 'int4', value: bucketDataRows }
+          ]
+        });
+        await factory.db.query({
+          statement: `
+            INSERT INTO bucket_parameters (group_id, source_table, source_key, lookup, bucket_parameters)
+            SELECT $1, 'test', int4send(i), ''::bytea, '[]' FROM generate_series(1, 10) i
+          `,
+          params: [{ type: 'int4', value: gid }]
+        });
+        await factory.db.query({
+          statement: `
+            INSERT INTO ${currentDataTable} (group_id, source_table, source_key, buckets, data, lookups)
+            SELECT $1, 'test', int4send(i), '[]', ''::bytea, '{}' FROM generate_series(1, 10) i
+          `,
+          params: [{ type: 'int4', value: gid }]
+        });
+        await factory.db.query({
+          statement: `
+            INSERT INTO source_tables (id, group_id, connection_id, schema_name, table_name)
+            VALUES ($2, $1, 1, 'public', 'test')
+          `,
+          params: [
+            { type: 'int4', value: gid },
+            { type: 'varchar', value: `test-${gid}` }
+          ]
+        });
+      };
+      await seed(groupId, CLEAR_BATCH_LIMIT + 100);
+      await seed(otherGroupId, 10);
+
+      const countRows = async (table: string, gid: number) => {
+        const [row] = await factory.db.queryRows<{ count: bigint }>({
+          statement: `SELECT COUNT(*) AS count FROM ${table} WHERE group_id = $1`,
+          params: [{ type: 'int4', value: gid }]
+        });
+        return Number(row.count);
+      };
+
+      await bucketStorage.clear();
+
+      for (const table of ['bucket_data', 'bucket_parameters', currentDataTable, 'source_tables']) {
+        expect(await countRows(table, groupId), table).toEqual(0);
+      }
+      // Rows of other groups are not affected.
+      expect(await countRows('bucket_data', otherGroupId)).toEqual(10);
+      expect(await countRows('bucket_parameters', otherGroupId)).toEqual(10);
+      expect(await countRows(currentDataTable, otherGroupId)).toEqual(10);
+      expect(await countRows('source_tables', otherGroupId)).toEqual(1);
+
+      // An aborted signal stops the operation.
+      await expect(bucketStorage.clear({ signal: AbortSignal.abort() })).rejects.toThrow(
+        framework.ReplicationAbortedError
+      );
+    });
+
     /**
      * The split of returned results can vary depending on storage drivers.
      * The large rows here are 2MB large while the default chunk limit is 1mb.
@@ -39,7 +260,8 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         )
       );
       const bucketStorage = factory.getInstance(syncRules);
-      const globalBucket = bucketRequest(syncRules, 'global[]');
+      const syncRulesContent = syncRules.syncConfigContent[0];
+      const globalBucket = bucketRequest(syncRulesContent, 'global[]');
 
       const result = await (async () => {
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -94,7 +316,9 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
 
       const options: storage.BucketDataBatchOptions = {};
 
-      const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options));
+      const batch1 = await test_utils.getBatchArray(
+        bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [globalBucket], options)
+      );
       expect(test_utils.getBatchData(batch1)).toEqual([
         { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
       ]);
@@ -104,9 +328,9 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         next_after: '1'
       });
 
-      const batch2 = await test_utils.fromAsync(
+      const batch2 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
-          checkpoint,
+          test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
           options
         )
@@ -120,9 +344,9 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         next_after: '2'
       });
 
-      const batch3 = await test_utils.fromAsync(
+      const batch3 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
-          checkpoint,
+          test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
           options
         )
@@ -136,9 +360,9 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         next_after: '3'
       });
 
-      const batch4 = await test_utils.fromAsync(
+      const batch4 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
-          checkpoint,
+          test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],
           options
         )

package/test/src/storage_compacting.test.ts

@@ -1,5 +1,5 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
@@ -17,6 +17,7 @@ bucket_definitions:
       `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     const result = await (async () => {
       await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -43,13 +44,15 @@ bucket_definitions:
 
     // Compact with an explicit bucket name — exercises the this.buckets
     // iteration path, NOT the compactAllBuckets discovery path.
-    await bucketStorage.compact({
-      compactBuckets: [bucketRequest(syncRules, 'global[]').bucket],
+    await compactActive(factory, {
+      compactBuckets: [bucketRequest(syncRulesContent, 'global[]').bucket],
       minBucketChanges: 1
     });
 
-    const batch = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
+    const batch = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [
+        bucketRequest(syncRulesContent, 'global[]', 0n)
+      ])
     );
 
     expect(batch.chunkData.data).toMatchObject([
@@ -70,7 +73,8 @@ bucket_definitions:
       `)
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const request = bucketRequest(syncRules, 'global[]');
+    const syncRulesContent = syncRules.syncConfigContent[0];
+    const request = bucketRequest(syncRulesContent, 'global[]');
 
     const result = await (async () => {
       await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -106,18 +110,22 @@ bucket_definitions:
     })();
 
     const checkpoint = result!.flushed_op;
-    const rowsBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const rowsBefore = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataBefore = test_utils.getBatchData(rowsBefore);
     const clearToOpId = BigInt(dataBefore[2].op_id);
 
-    const compactor = new PostgresCompactor(factory.db, bucketStorage.group_id, {});
+    const compactor = new PostgresCompactor(factory.db, bucketStorage.replicationStreamId, {});
     // Trigger the private method directly
     await expect(compactor.clearBucketForTests(request.bucket, clearToOpId)).rejects.toThrow(
       /Unexpected PUT operation/
     );
 
     // The method wraps in a transaction; on assertion error the bucket must remain unchanged.
-    const rowsAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const rowsAfter = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     expect(test_utils.getBatchData(rowsAfter)).toEqual(dataBefore);
   });
 });

package/test/src/storage_sync.test.ts

@@ -17,6 +17,50 @@ function registerStorageVersionTests(storageVersion: number) {
       tableIdStrings: storageFactory.tableIdStrings
     });
 
+    test('updates source metadata on an existing resolved table', async () => {
+      await using factory = await storageFactory.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id FROM test
+`,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const source: storage.SourceEntityDescriptor = {
+        connectionTag: storage.SourceTable.DEFAULT_TAG,
+        objectId: 'test',
+        schema: 'public',
+        name: 'test',
+        replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }]
+      };
+
+      const initial = await writer.resolveTables({ connection_id: 1, source });
+      expect(initial.tables[0].sourceMetadata).toBeNull();
+
+      const sourceMetadata = { captureTableObjectId: 42 };
+      const updated = await writer.resolveTables({
+        connection_id: 1,
+        source,
+        reconcileSourceTables: ({ candidates }) => ({
+          compatibleTables: candidates.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
+          incompatibleTables: [],
+          newTableValues: { sourceMetadata }
+        })
+      });
+
+      expect(updated.tables.map((table) => table.id.toString())).toEqual(
+        initial.tables.map((table) => table.id.toString())
+      );
+      expect(updated.tables[0].sourceMetadata).toEqual(sourceMetadata);
+      expect((await writer.getSourceTableStatus(updated.tables[0]))?.sourceMetadata).toEqual(sourceMetadata);
+    });
+
     test('large batch (2)', async () => {
       // Test syncing a batch of data that is small in count,
       // but large enough in size to be split over multiple returned chunks.
@@ -34,7 +78,8 @@ function registerStorageVersionTests(storageVersion: number) {
         )
       );
       const bucketStorage = factory.getInstance(syncRules);
-      const globalBucket = bucketRequest(syncRules, 'global[]');
+      const syncRulesContent = syncRules.syncConfigContent[0];
+      const globalBucket = bucketRequest(syncRulesContent, 'global[]');
 
       await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
 
@@ -89,7 +134,9 @@ function registerStorageVersionTests(storageVersion: number) {
 
       const options: storage.BucketDataBatchOptions = {};
 
-      const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options));
+      const batch1 = await test_utils.getBatchArray(
+        bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [globalBucket], options)
+      );
       expect(test_utils.getBatchData(batch1)).toEqual([
         { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
       ]);
@@ -99,9 +146,9 @@ function registerStorageVersionTests(storageVersion: number) {
         next_after: '1'
       });
 
-      const batch2 = await test_utils.fromAsync(
+      const batch2 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
-          checkpoint,
+          test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
           options
         )
@@ -115,9 +162,9 @@ function registerStorageVersionTests(storageVersion: number) {
         next_after: '2'
       });
 
-      const batch3 = await test_utils.fromAsync(
+      const batch3 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
-          checkpoint,
+          test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
           options
         )
@@ -131,9 +178,9 @@ function registerStorageVersionTests(storageVersion: number) {
         next_after: '3'
       });
 
-      const batch4 = await test_utils.fromAsync(
+      const batch4 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
-          checkpoint,
+          test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],
           options
         )

package/test/src/util.ts

@@ -36,4 +36,4 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({
 export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP;
 export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory;
 
-export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;
+export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS.filter((v) => v < 3);

package/test/tsconfig.json

@@ -1,9 +1,7 @@
 {
   "extends": "../../../tsconfig.tests.json",
   "compilerOptions": {
-    "declarationDir": "dist/@types",
-    "tsBuildInfoFile": "dist/.tsbuildinfo",
-    "lib": ["ES2022", "esnext.disposable"],
+    "lib": ["ES2024", "esnext.disposable"],
     "rootDir": "src"
   },
   "include": ["src"],

package/tsconfig.json

@@ -1,11 +1,8 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist",
-    "declarationDir": "dist/@types",
-    "tsBuildInfoFile": "dist/.tsbuildinfo",
     "rootDir": "src",
-    "skipLibCheck": true
+    "outDir": "dist"
   },
   "include": ["src"],
   "references": [