@powersync/service-core-tests 0.11.0 → 0.12.0

package/src/tests/register-compacting-tests.ts

@@ -6,14 +6,15 @@ const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
 
 export function registerCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting (1)', async () => {
-    const sync_rules = test_utils.testRules(`
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data: [select * from test]
-    `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    `
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
@@ -42,6 +43,8 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('t2')
       });
+
+      await batch.commit('1/1');
     });
 
     const checkpoint = result!.flushed_op;
@@ -72,6 +75,7 @@ bucket_definitions:
         op_id: '3'
       }
     ]);
+    expect(batchBefore.targetOp).toEqual(null);
 
     await bucketStorage.compact({
       clearBatchLimit: 2,
@@ -84,6 +88,8 @@ bucket_definitions:
     );
     const dataAfter = batchAfter.chunkData.data;
     const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['global[]']);
+    bucketStorage.clearChecksumCache();
+    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
@@ -106,20 +112,22 @@ bucket_definitions:
       }
     ]);
 
-    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
+    expect(checksumAfter.get('global[]')).toEqual(checksumBefore.get('global[]'));
+    expect(checksumAfter2.get('global[]')).toEqual(checksumBefore.get('global[]'));
 
     test_utils.validateCompactedBucket(dataBefore, dataAfter);
   });
 
   test('compacting (2)', async () => {
-    const sync_rules = test_utils.testRules(`
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data: [select * from test]
-    `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    `
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
@@ -157,6 +165,8 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('t2')
       });
+
+      await batch.commit('1/1');
     });
 
     const checkpoint = result!.flushed_op;
@@ -204,6 +214,7 @@ bucket_definitions:
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
     );
     const dataAfter = batchAfter.chunkData.data;
+    bucketStorage.clearChecksumCache();
     const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(4n);
@@ -220,20 +231,24 @@ bucket_definitions:
         op_id: '4'
       }
     ]);
-    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
+    expect(checksumAfter.get('global[]')).toEqual({
+      ...checksumBefore.get('global[]'),
+      count: 2
+    });
 
     test_utils.validateCompactedBucket(dataBefore, dataAfter);
   });
 
   test('compacting (3)', async () => {
-    const sync_rules = test_utils.testRules(`
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data: [select * from test]
-    `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    `
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
@@ -262,6 +277,8 @@ bucket_definitions:
         },
         beforeReplicaId: 't1'
       });
+
+      await batch.commit('1/1');
     });
 
     const checkpoint1 = result!.flushed_op;
@@ -276,6 +293,7 @@ bucket_definitions:
         },
         beforeReplicaId: 't2'
       });
+      await batch.commit('2/1');
     });
     const checkpoint2 = result2!.flushed_op;
 
@@ -289,6 +307,7 @@ bucket_definitions:
       bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', 0n]]))
     );
     const dataAfter = batchAfter.chunkData.data;
+    await bucketStorage.clearChecksumCache();
     const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(4n);
@@ -307,18 +326,18 @@ bucket_definitions:
   });
 
   test('compacting (4)', async () => {
-    const sync_rules = test_utils.testRules(/* yaml */
-    ` bucket_definitions:
-        grouped:
-          # The parameter query here is not important
-          # We specifically don't want to create bucket_parameter records here
-          # since the op_ids for bucket_data could vary between storage implementations.
-          parameters: select 'b' as b
-          data:
-            - select * from test where b = bucket.b`);
-
     await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    const syncRules = await factory.updateSyncRules({
+      /* yaml */ content: ` bucket_definitions:
+          grouped:
+            # The parameter query here is not important
+            # We specifically don't want to create bucket_parameter records here
+            # since the op_ids for bucket_data could vary between storage implementations.
+            parameters: select 'b' as b
+            data:
+              - select * from test where b = bucket.b`
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       /**
@@ -383,6 +402,8 @@ bucket_definitions:
           },
           afterReplicaId: test_utils.rid('t2')
         });
+
+        await batch.commit('1/1');
       }
     });
 
@@ -431,4 +452,73 @@ bucket_definitions:
       ])
     );
   });
+
+  test('partial checksums after compacting', async () => {
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `
+    });
+    const bucketStorage = factory.getInstance(syncRules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1'
+        },
+        afterReplicaId: 't1'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't2'
+        },
+        afterReplicaId: 't2'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.DELETE,
+        before: {
+          id: 't1'
+        },
+        beforeReplicaId: 't1'
+      });
+
+      await batch.commit('1/1');
+    });
+
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
+
+    const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.DELETE,
+        before: {
+          id: 't2'
+        },
+        beforeReplicaId: 't2'
+      });
+      await batch.commit('2/1');
+    });
+    const checkpoint2 = result2!.flushed_op;
+    await bucketStorage.clearChecksumCache();
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, ['global[]']);
+    expect(checksumAfter.get('global[]')).toEqual({
+      bucket: 'global[]',
+      count: 4,
+      checksum: 1874612650
+    });
+  });
 }

package/src/tests/register-data-storage-tests.ts

@@ -3,11 +3,13 @@ import {
   getUuidReplicaIdentityBson,
   InternalOpId,
   OplogEntry,
+  SaveOptions,
   storage
 } from '@powersync/service-core';
-import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules';
+import { DateTimeValue, ParameterLookup, RequestParameters } from '@powersync/service-sync-rules';
 import { expect, test, describe, beforeEach } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
+import { SqlBucketDescriptor } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js';
 
 export const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
 
@@ -411,7 +413,7 @@ bucket_definitions:
 
     const parameters = new RequestParameters({ sub: 'u1' }, {});
 
-    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
+    const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0];
 
     const lookups = q1.getLookups(parameters);
     expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]);
@@ -419,12 +421,16 @@ bucket_definitions:
     const parameter_sets = await checkpoint.getParameterSets(lookups);
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
-      getParameterSets(lookups) {
-        return checkpoint.getParameterSets(lookups);
-      }
-    });
-    expect(buckets).toEqual([{ bucket: 'by_workspace["workspace1"]', priority: 3 }]);
+    const buckets = await sync_rules
+      .getBucketParameterQuerier(test_utils.querierOptions(parameters))
+      .querier.queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return checkpoint.getParameterSets(lookups);
+        }
+      });
+    expect(buckets).toEqual([
+      { bucket: 'by_workspace["workspace1"]', priority: 3, definition: 'by_workspace', inclusion_reasons: ['default'] }
+    ]);
   });
 
   test('save and load parameters with dynamic global buckets', async () => {
@@ -482,7 +488,7 @@ bucket_definitions:
 
     const parameters = new RequestParameters({ sub: 'unknown' }, {});
 
-    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
+    const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0];
 
     const lookups = q1.getLookups(parameters);
     expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]);
@@ -491,15 +497,27 @@ bucket_definitions:
     parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
 
-    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
-      getParameterSets(lookups) {
-        return checkpoint.getParameterSets(lookups);
-      }
-    });
+    const buckets = await sync_rules
+      .getBucketParameterQuerier(test_utils.querierOptions(parameters))
+      .querier.queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return checkpoint.getParameterSets(lookups);
+        }
+      });
     buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
     expect(buckets).toEqual([
-      { bucket: 'by_public_workspace["workspace1"]', priority: 3 },
-      { bucket: 'by_public_workspace["workspace3"]', priority: 3 }
+      {
+        bucket: 'by_public_workspace["workspace1"]',
+        priority: 3,
+        definition: 'by_public_workspace',
+        inclusion_reasons: ['default']
+      },
+      {
+        bucket: 'by_public_workspace["workspace3"]',
+        priority: 3,
+        definition: 'by_public_workspace',
+        inclusion_reasons: ['default']
+      }
     ]);
   });
 
@@ -573,7 +591,7 @@ bucket_definitions:
     const parameters = new RequestParameters({ sub: 'u1' }, {});
 
     // Test intermediate values - could be moved to sync_rules.test.ts
-    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
+    const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0];
     const lookups1 = q1.getLookups(parameters);
     expect(lookups1).toEqual([ParameterLookup.normalized('by_workspace', '1', [])]);
 
@@ -581,7 +599,7 @@ bucket_definitions:
     parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const q2 = sync_rules.bucketDescriptors[0].parameterQueries[1];
+    const q2 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[1];
     const lookups2 = q2.getLookups(parameters);
     expect(lookups2).toEqual([ParameterLookup.normalized('by_workspace', '2', ['u1'])]);
 
@@ -591,11 +609,13 @@ bucket_definitions:
 
     // Test final values - the important part
     const buckets = (
-      await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
-        getParameterSets(lookups) {
-          return checkpoint.getParameterSets(lookups);
-        }
-      })
+      await sync_rules
+        .getBucketParameterQuerier(test_utils.querierOptions(parameters))
+        .querier.queryDynamicBucketDescriptions({
+          getParameterSets(lookups) {
+            return checkpoint.getParameterSets(lookups);
+          }
+        })
     ).map((e) => e.bucket);
     buckets.sort();
     expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']);
@@ -1976,4 +1996,70 @@ bucket_definitions:
     // we expect 0n and 1n, or 1n and 2n.
     expect(checkpoint2).toBeGreaterThan(checkpoint1);
   });
+
+  test('data with custom types', async () => {
+    await using factory = await generateStorageFactory();
+    const testValue = {
+      sourceTable: TEST_TABLE,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 't1',
+        description: new DateTimeValue('2025-08-28T11:30:00')
+      },
+      afterReplicaId: test_utils.rid('t1')
+    } satisfies SaveOptions;
+
+    {
+      // First, deploy old sync rules and row with date time value
+      const syncRules = await factory.updateSyncRules({
+        content: `
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id, description FROM test
+  `
+      });
+      const bucketStorage = factory.getInstance(syncRules);
+      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        await batch.save(testValue);
+        await batch.commit('1/1');
+      });
+
+      const { checkpoint } = await bucketStorage.getCheckpoint();
+      const batch = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
+      );
+      expect(batch[0].chunkData.data).toMatchObject([
+        {
+          data: '{"id":"t1","description":"2025-08-28 11:30:00"}'
+        }
+      ]);
+    }
+
+    const syncRules = await factory.updateSyncRules({
+      content: `
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id, description FROM test
+  
+  config:
+    edition: 2
+  `
+    });
+    const bucketStorage = factory.getInstance(syncRules);
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save(testValue);
+      await batch.commit('1/2');
+    });
+    const { checkpoint } = await bucketStorage.getCheckpoint();
+    const batch = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['2#global[]', 0n]]))
+    );
+    expect(batch[0].chunkData.data).toMatchObject([
+      {
+        data: '{"id":"t1","description":"2025-08-28T11:30:00"}'
+      }
+    ]);
+  });
 }