@powersync/service-module-postgres-storage 0.16.3 → 0.18.0

package/test/src/storage.test.ts

@@ -1,8 +1,14 @@
-import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { framework, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
+import * as t from 'ts-codec';
 import { describe, expect, test } from 'vitest';
+import { CLEAR_BATCH_LIMIT } from '../../src/storage/PostgresSyncRulesStorage.js';
 import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
+const CheckpointRequestedAtRow = t.object({
+  checkpoint_requested_at: t.Null.or(framework.codecs.date)
+});
+
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 
 for (let storageVersion of TEST_STORAGE_VERSIONS) {
@@ -20,6 +26,209 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
     register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
 
   describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => {
+    test('uses checkpoint_requested_at as the client-requested checkpoint marker', async () => {
+      await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data: []
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const requestedAt = async (userId = 'user1') =>
+        (
+          await factory.db.sql`
+            SELECT
+              checkpoint_requested_at
+            FROM
+              write_checkpoints
+            WHERE
+              user_id = ${{ type: 'varchar', value: userId }}
+          `
+            .decoded(CheckpointRequestedAtRow)
+            .first()
+        )?.checkpoint_requested_at;
+
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '5/0' }, checkpoint_request_id: 42n }
+      ]);
+      const requested = await requestedAt();
+      expect(requested).toBeInstanceOf(Date);
+
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 41n }
+      ]);
+      await expect(requestedAt()).resolves.toEqual(requested);
+
+      const expiredRequestedAt = new Date('2024-01-01T00:00:00.000Z');
+      await factory.db.sql`
+        UPDATE write_checkpoints
+        SET
+          checkpoint_requested_at = ${{ type: 1184, value: expiredRequestedAt.toISOString() }}
+        WHERE
+          user_id = 'user1'
+      `.execute();
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 42n }
+      ]);
+      // Retrying the current id refreshes its retention timestamp. The shared
+      // checkpoint tests verify that the original source head is preserved.
+      expect((await requestedAt())!.getTime()).toBeGreaterThan(expiredRequestedAt.getTime());
+
+      await factory.db.sql`
+        UPDATE write_checkpoints
+        SET
+          checkpoint_requested_at = ${{ type: 1184, value: expiredRequestedAt.toISOString() }}
+        WHERE
+          user_id = 'user1'
+      `.execute();
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 43n }
+      ]);
+      // A greater id refreshes retention while advancing the stored checkpoint.
+      expect((await requestedAt())!.getTime()).toBeGreaterThan(expiredRequestedAt.getTime());
+
+      await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '7/0' } }]);
+      await expect(requestedAt()).resolves.toBeNull();
+
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user2', heads: { '1': '8/0' }, checkpoint_request_id: 50n }
+      ]);
+      await factory.db.sql`
+        UPDATE write_checkpoints
+        SET
+          checkpoint_requested_at = ${{ type: 1184, value: '2024-01-01T00:00:00.000Z' }}
+        WHERE
+          user_id = 'user2'
+      `.execute();
+      await bucketStorage.compact({
+        compactBuckets: [],
+        deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z')
+      });
+      // Compaction removes expired requests based on the refreshed timestamp.
+      await expect(requestedAt('user2')).resolves.toBeUndefined();
+
+      const customRequestedAt = async (userId = 'custom1') =>
+        (
+          await factory.db.sql`
+            SELECT
+              checkpoint_requested_at
+            FROM
+              custom_write_checkpoints
+            WHERE
+              user_id = ${{ type: 'varchar', value: userId }}
+          `
+            .decoded(CheckpointRequestedAtRow)
+            .first()
+        )?.checkpoint_requested_at;
+
+      bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      await writer.markAllSnapshotDone('1/1');
+      const customCheckpointRequestedAt = new Date('2024-01-01T00:00:00.000Z');
+      writer.addCustomWriteCheckpoint({
+        user_id: 'custom1',
+        checkpoint: 51n,
+        checkpoint_requested_at: customCheckpointRequestedAt
+      });
+      await writer.flush();
+      await expect(customRequestedAt()).resolves.toEqual(customCheckpointRequestedAt);
+
+      writer.addCustomWriteCheckpoint({
+        user_id: 'custom1',
+        checkpoint: 52n
+      });
+      await writer.flush();
+      await expect(customRequestedAt()).resolves.toBeNull();
+    });
+
+    test('clears storage in batches', async () => {
+      await using factory = await POSTGRES_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data: []
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const groupId = bucketStorage.replicationStreamId;
+      const otherGroupId = groupId + 1;
+      const currentDataTable = bucketStorage.storageConfig.softDeleteCurrentData ? 'v3_current_data' : 'current_data';
+
+      // Seed more than one batch of bucket data, plus rows in the other tables,
+      // directly - clear() only depends on group_id.
+      const seed = async (gid: number, bucketDataRows: number) => {
+        await factory.db.query({
+          statement: `
+            INSERT INTO bucket_data (group_id, bucket_name, op_id, op, checksum)
+            SELECT $1, 'global[]', i, 'PUT', 0 FROM generate_series(1, $2) i
+          `,
+          params: [
+            { type: 'int4', value: gid },
+            { type: 'int4', value: bucketDataRows }
+          ]
+        });
+        await factory.db.query({
+          statement: `
+            INSERT INTO bucket_parameters (group_id, source_table, source_key, lookup, bucket_parameters)
+            SELECT $1, 'test', int4send(i), ''::bytea, '[]' FROM generate_series(1, 10) i
+          `,
+          params: [{ type: 'int4', value: gid }]
+        });
+        await factory.db.query({
+          statement: `
+            INSERT INTO ${currentDataTable} (group_id, source_table, source_key, buckets, data, lookups)
+            SELECT $1, 'test', int4send(i), '[]', ''::bytea, '{}' FROM generate_series(1, 10) i
+          `,
+          params: [{ type: 'int4', value: gid }]
+        });
+        await factory.db.query({
+          statement: `
+            INSERT INTO source_tables (id, group_id, connection_id, schema_name, table_name)
+            VALUES ($2, $1, 1, 'public', 'test')
+          `,
+          params: [
+            { type: 'int4', value: gid },
+            { type: 'varchar', value: `test-${gid}` }
+          ]
+        });
+      };
+      await seed(groupId, CLEAR_BATCH_LIMIT + 100);
+      await seed(otherGroupId, 10);
+
+      const countRows = async (table: string, gid: number) => {
+        const [row] = await factory.db.queryRows<{ count: bigint }>({
+          statement: `SELECT COUNT(*) AS count FROM ${table} WHERE group_id = $1`,
+          params: [{ type: 'int4', value: gid }]
+        });
+        return Number(row.count);
+      };
+
+      await bucketStorage.clear();
+
+      for (const table of ['bucket_data', 'bucket_parameters', currentDataTable, 'source_tables']) {
+        expect(await countRows(table, groupId), table).toEqual(0);
+      }
+      // Rows of other groups are not affected.
+      expect(await countRows('bucket_data', otherGroupId)).toEqual(10);
+      expect(await countRows('bucket_parameters', otherGroupId)).toEqual(10);
+      expect(await countRows(currentDataTable, otherGroupId)).toEqual(10);
+      expect(await countRows('source_tables', otherGroupId)).toEqual(1);
+
+      // An aborted signal stops the operation.
+      await expect(bucketStorage.clear({ signal: AbortSignal.abort() })).rejects.toThrow(
+        framework.ReplicationAbortedError
+      );
+    });
+
     /**
      * The split of returned results can vary depending on storage drivers.
      * The large rows here are 2MB large while the default chunk limit is 1mb.
@@ -99,7 +308,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
 
       const options: storage.BucketDataBatchOptions = {};
 
-      const batch1 = await test_utils.fromAsync(
+      const batch1 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [globalBucket], options)
       );
       expect(test_utils.getBatchData(batch1)).toEqual([
@@ -111,7 +320,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         next_after: '1'
       });
 
-      const batch2 = await test_utils.fromAsync(
+      const batch2 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
           test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
@@ -127,7 +336,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         next_after: '2'
       });
 
-      const batch3 = await test_utils.fromAsync(
+      const batch3 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
           test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
@@ -143,7 +352,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
         next_after: '3'
       });
 
-      const batch4 = await test_utils.fromAsync(
+      const batch4 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
           test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],

package/test/src/storage_compacting.test.ts

@@ -49,7 +49,7 @@ bucket_definitions:
       minBucketChanges: 1
     });
 
-    const batch = await test_utils.oneFromAsync(
+    const batch = await test_utils.getSingleBatchItem(
       bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [
         bucketRequest(syncRulesContent, 'global[]', 0n)
       ])
@@ -110,7 +110,7 @@ bucket_definitions:
     })();
 
     const checkpoint = result!.flushed_op;
-    const rowsBefore = await test_utils.oneFromAsync(
+    const rowsBefore = await test_utils.getSingleBatchItem(
       bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
     );
     const dataBefore = test_utils.getBatchData(rowsBefore);
@@ -123,7 +123,7 @@ bucket_definitions:
     );
 
     // The method wraps in a transaction; on assertion error the bucket must remain unchanged.
-    const rowsAfter = await test_utils.oneFromAsync(
+    const rowsAfter = await test_utils.getSingleBatchItem(
       bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
     );
     expect(test_utils.getBatchData(rowsAfter)).toEqual(dataBefore);

package/test/src/storage_sync.test.ts

@@ -17,6 +17,50 @@ function registerStorageVersionTests(storageVersion: number) {
       tableIdStrings: storageFactory.tableIdStrings
     });
 
+    test('updates source metadata on an existing resolved table', async () => {
+      await using factory = await storageFactory.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id FROM test
+`,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const source: storage.SourceEntityDescriptor = {
+        connectionTag: storage.SourceTable.DEFAULT_TAG,
+        objectId: 'test',
+        schema: 'public',
+        name: 'test',
+        replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }]
+      };
+
+      const initial = await writer.resolveTables({ connection_id: 1, source });
+      expect(initial.tables[0].sourceMetadata).toBeNull();
+
+      const sourceMetadata = { captureTableObjectId: 42 };
+      const updated = await writer.resolveTables({
+        connection_id: 1,
+        source,
+        reconcileSourceTables: ({ candidates }) => ({
+          compatibleTables: candidates.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
+          incompatibleTables: [],
+          newTableValues: { sourceMetadata }
+        })
+      });
+
+      expect(updated.tables.map((table) => table.id.toString())).toEqual(
+        initial.tables.map((table) => table.id.toString())
+      );
+      expect(updated.tables[0].sourceMetadata).toEqual(sourceMetadata);
+      expect((await writer.getSourceTableStatus(updated.tables[0]))?.sourceMetadata).toEqual(sourceMetadata);
+    });
+
     test('large batch (2)', async () => {
       // Test syncing a batch of data that is small in count,
       // but large enough in size to be split over multiple returned chunks.
@@ -90,7 +134,7 @@ function registerStorageVersionTests(storageVersion: number) {
 
       const options: storage.BucketDataBatchOptions = {};
 
-      const batch1 = await test_utils.fromAsync(
+      const batch1 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [globalBucket], options)
       );
       expect(test_utils.getBatchData(batch1)).toEqual([
@@ -102,7 +146,7 @@ function registerStorageVersionTests(storageVersion: number) {
         next_after: '1'
       });
 
-      const batch2 = await test_utils.fromAsync(
+      const batch2 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
           test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
@@ -118,7 +162,7 @@ function registerStorageVersionTests(storageVersion: number) {
         next_after: '2'
       });
 
-      const batch3 = await test_utils.fromAsync(
+      const batch3 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
           test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
@@ -134,7 +178,7 @@ function registerStorageVersionTests(storageVersion: number) {
         next_after: '3'
       });
 
-      const batch4 = await test_utils.fromAsync(
+      const batch4 = await test_utils.getBatchArray(
         bucketStorage.getBucketDataBatch(
           test_utils.testCheckpoint(checkpoint),
           [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],

package/test/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "../../../tsconfig.tests.json",
   "compilerOptions": {
-    "lib": ["ES2022", "esnext.disposable"],
+    "lib": ["ES2024", "esnext.disposable"],
     "rootDir": "src"
   },
   "include": ["src"],