@powersync/service-core-tests 0.13.2 → 0.15.0

package/src/tests/register-compacting-tests.ts

@@ -1,22 +1,26 @@
-import { storage } from '@powersync/service-core';
+import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
+import { bucketRequest } from '../test-utils/test-utils-index.js';
+import { bucketRequestMap, bucketRequests } from './util.js';
 
-const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
+export function registerCompactTests(config: storage.TestStorageConfig) {
+  const generateStorageFactory = config.factory;
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
 
-export function registerCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting (1)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -49,27 +53,24 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const batchBefore = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
-    );
+    const request = bucketRequest(syncRules, 'global[]');
+
+    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
 
     expect(dataBefore).toMatchObject([
       {
-        checksum: 2634521662,
         object_id: 't1',
         op: 'PUT',
         op_id: '1'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
         op: 'PUT',
         op_id: '2'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
         op: 'PUT',
         op_id: '3'
@@ -85,53 +86,47 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
-    );
+    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataAfter = batchAfter.chunkData.data;
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
     bucketStorage.clearChecksumCache();
-    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, [request]);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
+      dataBefore[0],
       {
-        checksum: 2634521662,
-        object_id: 't1',
-        op: 'PUT',
-        op_id: '1'
-      },
-      {
-        checksum: 4243212114,
+        checksum: dataBefore[1].checksum,
         op: 'MOVE',
         op_id: '2'
       },
       {
-        checksum: 4243212114,
+        checksum: dataBefore[2].checksum,
         object_id: 't2',
         op: 'PUT',
         op_id: '3'
       }
     ]);
 
-    expect(checksumAfter.get('global[]')).toEqual(checksumBefore.get('global[]'));
-    expect(checksumAfter2.get('global[]')).toEqual(checksumBefore.get('global[]'));
+    expect(checksumAfter.get(request.bucket)).toEqual(checksumBefore.get(request.bucket));
+    expect(checksumAfter2.get(request.bucket)).toEqual(checksumBefore.get(request.bucket));
 
     test_utils.validateCompactedBucket(dataBefore, dataAfter);
   });
 
   test('compacting (2)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -172,37 +167,29 @@ bucket_definitions:
     });
 
     const checkpoint = result!.flushed_op;
+    const request = bucketRequest(syncRules, 'global[]');
 
-    const batchBefore = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
-    );
+    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
 
+    // op_id sequence depends on the storage implementation
     expect(dataBefore).toMatchObject([
       {
-        checksum: 2634521662,
         object_id: 't1',
-        op: 'PUT',
-        op_id: '1'
+        op: 'PUT'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
-        op: 'PUT',
-        op_id: '2'
+        op: 'PUT'
       },
       {
-        checksum: 4228978084,
         object_id: 't1',
-        op: 'REMOVE',
-        op_id: '3'
+        op: 'REMOVE'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
-        op: 'PUT',
-        op_id: '4'
+        op: 'PUT'
       }
     ]);
 
@@ -214,29 +201,28 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
-    );
+    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataAfter = batchAfter.chunkData.data;
     bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
 
-    expect(batchAfter.targetOp).toEqual(4n);
+    expect(batchAfter.targetOp).toBeLessThanOrEqual(checkpoint);
     expect(dataAfter).toMatchObject([
       {
-        checksum: -1778190028,
-        op: 'CLEAR',
-        op_id: '3'
+        checksum: addChecksums(
+          addChecksums(dataBefore[0].checksum as number, dataBefore[1].checksum as number),
+          dataBefore[2].checksum as number
+        ),
+        op: 'CLEAR'
       },
       {
-        checksum: 4243212114,
+        checksum: dataBefore[3].checksum,
         object_id: 't2',
-        op: 'PUT',
-        op_id: '4'
+        op: 'PUT'
       }
     ]);
-    expect(checksumAfter.get('global[]')).toEqual({
-      ...checksumBefore.get('global[]'),
+    expect(checksumAfter.get(request.bucket)).toEqual({
+      ...checksumBefore.get(request.bucket),
       count: 2
     });
 
@@ -245,16 +231,17 @@ bucket_definitions:
 
   test('compacting (3)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -286,7 +273,8 @@ bucket_definitions:
     });
 
     const checkpoint1 = result!.flushed_op;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint1, ['global[]']);
+    const request = bucketRequest(syncRules, 'global[]');
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint1, [request]);
 
     const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
@@ -309,43 +297,39 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', 0n]]))
-    );
+    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
     const dataAfter = batchAfter.chunkData.data;
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
 
-    expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
       {
-        checksum: 1874612650,
-        op: 'CLEAR',
-        op_id: '4'
+        op: 'CLEAR'
       }
     ]);
-    expect(checksumAfter.get('global[]')).toEqual({
-      bucket: 'global[]',
+    expect(checksumAfter.get(request.bucket)).toEqual({
+      bucket: request.bucket,
       count: 1,
-      checksum: 1874612650
+      checksum: dataAfter[0].checksum
     });
   });
 
   test('compacting (4)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      /* yaml */ content: ` bucket_definitions:
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(` bucket_definitions:
           grouped:
             # The parameter query here is not important
             # We specifically don't want to create bucket_parameter records here
             # since the op_ids for bucket_data could vary between storage implementations.
             parameters: select 'b' as b
             data:
-              - select * from test where b = bucket.b`
-    });
+              - select * from test where b = bucket.b`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       /**
        * Repeatedly create operations which fall into different buckets.
        * The bucket operations are purposely interleaved as the op_id increases.
@@ -426,7 +410,7 @@ bucket_definitions:
     const batchAfter = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([
+        bucketRequestMap(syncRules, [
           ['grouped["b1"]', 0n],
           ['grouped["b2"]', 0n]
         ])
@@ -463,16 +447,17 @@ bucket_definitions:
 
   test('partial checksums after compacting', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -523,27 +508,32 @@ bucket_definitions:
       await batch.commit('2/1');
     });
     const checkpoint2 = result2!.flushed_op;
+    const request = bucketRequest(syncRules, 'global[]');
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
-    expect(checksumAfter.get('global[]')).toEqual({
-      bucket: 'global[]',
-      count: 4,
-      checksum: 1874612650
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const globalChecksum = checksumAfter.get(request.bucket);
+    expect(globalChecksum).toMatchObject({
+      bucket: request.bucket,
+      count: 4
     });
+
+    // storage-specific checksum - just check that it does not change
+    expect(globalChecksum).toMatchSnapshot();
   });
 
   test('partial checksums after compacting (2)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -566,7 +556,7 @@ bucket_definitions:
     });
 
     // Get checksums here just to populate the cache
-    await bucketStorage.getChecksums(result!.flushed_op, ['global[]']);
+    await bucketStorage.getChecksums(result!.flushed_op, bucketRequests(syncRules, ['global[]']));
     const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -588,12 +578,72 @@ bucket_definitions:
     });
 
     const checkpoint2 = result2!.flushed_op;
+    const request = bucketRequest(syncRules, 'global[]');
     // Check that the checksum was correctly updated with the clear operation after having a cached checksum
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
-    expect(checksumAfter.get('global[]')).toMatchObject({
-      bucket: 'global[]',
-      count: 1,
-      checksum: -1481659821
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const globalChecksum = checksumAfter.get(request.bucket);
+    expect(globalChecksum).toMatchObject({
+      bucket: request.bucket,
+      count: 1
+    });
+    // storage-specific checksum - just check that it does not change
+    expect(globalChecksum).toMatchSnapshot();
+  });
+
+  test('defaults maxOpId to current checkpoint', async () => {
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+      `)
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+
+    const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.commit('1/1');
+    });
+
+    const checkpoint1 = result1!.flushed_op;
+
+    const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // This is flushed but not committed (does not advance the checkpoint)
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.UPDATE,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+    });
+    const checkpoint2 = result2!.flushed_op;
+
+    const checkpointBeforeCompact = await bucketStorage.getCheckpoint();
+    expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1);
+
+    // With default options, Postgres compaction should use the active checkpoint.
+    await bucketStorage.compact({
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1,
+      minBucketChanges: 1,
+      minChangeRatio: 0
     });
+
+    const batchAfterDefaultCompact = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
+
+    // Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint.
+    expect(batchAfterDefaultCompact.chunkData.data).toMatchObject([
+      { op_id: '1', op: 'PUT', object_id: 't1' },
+      { op_id: '2', op: 'PUT', object_id: 't1' }
+    ]);
   });
 }

package/src/tests/register-data-storage-checkpoint-tests.ts

@@ -1,4 +1,4 @@
-import { storage } from '@powersync/service-core';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
 
@@ -12,17 +12,25 @@ import * as test_utils from '../test-utils/test-utils-index.js';
  *
  * ```
  */
-export function registerDataStorageCheckpointTests(generateStorageFactory: storage.TestStorageFactory) {
+export function registerDataStorageCheckpointTests(config: storage.TestStorageConfig) {
+  const generateStorageFactory = config.factory;
+  const storageVersion = config.storageVersion;
+
   test('managed write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
     `,
-      validate: false
-    });
+        {
+          validate: false,
+          storageVersion
+        }
+      )
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
 
     const abortController = new AbortController();
@@ -31,6 +39,10 @@ bucket_definitions:
       .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
       [Symbol.asyncIterator]();
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
       heads: { '1': '5/0' },
       user_id: 'user1'
@@ -55,16 +67,25 @@ bucket_definitions:
 
   test('managed write checkpoints - write after checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
     `,
-      validate: false
-    });
+        {
+          validate: false,
+          storageVersion
+        }
+      )
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
@@ -117,17 +138,26 @@ bucket_definitions:
 
   test('custom write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
     `,
-      validate: false
-    });
+        {
+          validate: false,
+          storageVersion
+        }
+      )
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
@@ -157,17 +187,26 @@ bucket_definitions:
 
   test('custom write checkpoints - standalone checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
     `,
-      validate: false
-    });
+        {
+          validate: false,
+          storageVersion
+        }
+      )
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
@@ -200,17 +239,26 @@ bucket_definitions:
 
   test('custom write checkpoints - write after checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
     `,
-      validate: false
-    });
+        {
+          validate: false,
+          storageVersion
+        }
+      )
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage