@powersync/service-core-tests 0.0.0-dev-20260203155513 → 0.0.0-dev-20260223082111

package/src/tests/register-compacting-tests.ts

@@ -1,19 +1,20 @@
-import { storage } from '@powersync/service-core';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
+import { bucketRequest, bucketRequestMap, bucketRequests } from './util.js';
 
 const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
 
 export function registerCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting (1)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -50,10 +51,10 @@ bucket_definitions:
     const checkpoint = result!.flushed_op;
 
     const batchBefore = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
     );
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
 
     expect(dataBefore).toMatchObject([
       {
@@ -81,16 +82,17 @@ bucket_definitions:
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
-      minBucketChanges: 1
+      minBucketChanges: 1,
+      minChangeRatio: 0
     });
 
     const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
     );
     const dataAfter = batchAfter.chunkData.data;
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
     bucketStorage.clearChecksumCache();
-    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
@@ -113,21 +115,25 @@ bucket_definitions:
       }
     ]);
 
-    expect(checksumAfter.get('global[]')).toEqual(checksumBefore.get('global[]'));
-    expect(checksumAfter2.get('global[]')).toEqual(checksumBefore.get('global[]'));
+    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual(
+      checksumBefore.get(bucketRequest(syncRules, 'global[]'))
+    );
+    expect(checksumAfter2.get(bucketRequest(syncRules, 'global[]'))).toEqual(
+      checksumBefore.get(bucketRequest(syncRules, 'global[]'))
+    );
 
     test_utils.validateCompactedBucket(dataBefore, dataAfter);
   });
 
   test('compacting (2)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -173,10 +179,10 @@ bucket_definitions:
     const checkpoint = result!.flushed_op;
 
     const batchBefore = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
     );
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
 
     expect(dataBefore).toMatchObject([
       {
@@ -209,15 +215,16 @@ bucket_definitions:
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
-      minBucketChanges: 1
+      minBucketChanges: 1,
+      minChangeRatio: 0
     });
 
     const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
     );
     const dataAfter = batchAfter.chunkData.data;
     bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
 
     expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
@@ -233,8 +240,8 @@ bucket_definitions:
         op_id: '4'
       }
     ]);
-    expect(checksumAfter.get('global[]')).toEqual({
-      ...checksumBefore.get('global[]'),
+    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual({
+      ...checksumBefore.get(bucketRequest(syncRules, 'global[]')),
       count: 2
     });
 
@@ -243,13 +250,13 @@ bucket_definitions:
 
   test('compacting (3)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -284,7 +291,7 @@ bucket_definitions:
     });
 
     const checkpoint1 = result!.flushed_op;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint1, ['global[]']);
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint1, bucketRequests(syncRules, ['global[]']));
 
     const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
@@ -303,15 +310,16 @@ bucket_definitions:
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
-      minBucketChanges: 1
+      minBucketChanges: 1,
+      minChangeRatio: 0
     });
 
     const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]]))
     );
     const dataAfter = batchAfter.chunkData.data;
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, bucketRequests(syncRules, ['global[]']));
 
     expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
@@ -321,8 +329,8 @@ bucket_definitions:
         op_id: '4'
       }
     ]);
-    expect(checksumAfter.get('global[]')).toEqual({
-      bucket: 'global[]',
+    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual({
+      bucket: bucketRequest(syncRules, 'global[]'),
       count: 1,
       checksum: 1874612650
     });
@@ -330,16 +338,16 @@ bucket_definitions:
 
   test('compacting (4)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      /* yaml */ content: ` bucket_definitions:
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(` bucket_definitions:
           grouped:
             # The parameter query here is not important
             # We specifically don't want to create bucket_parameter records here
             # since the op_ids for bucket_data could vary between storage implementations.
             parameters: select 'b' as b
             data:
-              - select * from test where b = bucket.b`
-    });
+              - select * from test where b = bucket.b`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -416,13 +424,14 @@ bucket_definitions:
       clearBatchLimit: 100,
       moveBatchLimit: 100,
       moveBatchQueryLimit: 100, // Larger limit for a larger window of operations
-      minBucketChanges: 1
+      minBucketChanges: 1,
+      minChangeRatio: 0
     });
 
     const batchAfter = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([
+        bucketRequestMap(syncRules, [
           ['grouped["b1"]', 0n],
           ['grouped["b2"]', 0n]
         ])
@@ -459,13 +468,13 @@ bucket_definitions:
 
   test('partial checksums after compacting', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -503,7 +512,8 @@ bucket_definitions:
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
-      minBucketChanges: 1
+      minBucketChanges: 1,
+      minChangeRatio: 0
     });
 
     const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -519,9 +529,9 @@ bucket_definitions:
     });
     const checkpoint2 = result2!.flushed_op;
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
-    expect(checksumAfter.get('global[]')).toEqual({
-      bucket: 'global[]',
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, bucketRequests(syncRules, ['global[]']));
+    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual({
+      bucket: bucketRequest(syncRules, 'global[]'),
       count: 4,
       checksum: 1874612650
     });
@@ -529,13 +539,13 @@ bucket_definitions:
 
   test('partial checksums after compacting (2)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data: [select * from test]
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -561,7 +571,7 @@ bucket_definitions:
     });
 
     // Get checksums here just to populate the cache
-    await bucketStorage.getChecksums(result!.flushed_op, ['global[]']);
+    await bucketStorage.getChecksums(result!.flushed_op, bucketRequests(syncRules, ['global[]']));
     const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -578,14 +588,15 @@ bucket_definitions:
       clearBatchLimit: 20,
       moveBatchLimit: 10,
       moveBatchQueryLimit: 10,
-      minBucketChanges: 1
+      minBucketChanges: 1,
+      minChangeRatio: 0
     });
 
     const checkpoint2 = result2!.flushed_op;
     // Check that the checksum was correctly updated with the clear operation after having a cached checksum
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
-    expect(checksumAfter.get('global[]')).toMatchObject({
-      bucket: 'global[]',
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, bucketRequests(syncRules, ['global[]']));
+    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toMatchObject({
+      bucket: bucketRequest(syncRules, 'global[]'),
       count: 1,
       checksum: -1481659821
     });

package/src/tests/register-data-storage-checkpoint-tests.ts

@@ -1,4 +1,4 @@
-import { storage } from '@powersync/service-core';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
 
@@ -15,14 +15,13 @@ import * as test_utils from '../test-utils/test-utils-index.js';
 export function registerDataStorageCheckpointTests(generateStorageFactory: storage.TestStorageFactory) {
   test('managed write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     data: []
-    `,
-      validate: false
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
 
     const abortController = new AbortController();
@@ -55,14 +54,13 @@ bucket_definitions:
 
   test('managed write checkpoints - write after checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     data: []
-    `,
-      validate: false
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
 
     const abortController = new AbortController();
@@ -117,14 +115,13 @@ bucket_definitions:
 
   test('custom write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     data: []
-    `,
-      validate: false
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
@@ -157,14 +154,13 @@ bucket_definitions:
 
   test('custom write checkpoints - standalone checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     data: []
-    `,
-      validate: false
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
@@ -200,14 +196,13 @@ bucket_definitions:
 
   test('custom write checkpoints - write after checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules({
-      content: `
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     data: []
-    `,
-      validate: false
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);