@powersync/service-core-tests 0.10.4 → 0.12.0

package/src/tests/register-data-storage-tests.ts

@@ -3,11 +3,13 @@ import {
   getUuidReplicaIdentityBson,
   InternalOpId,
   OplogEntry,
+  SaveOptions,
   storage
 } from '@powersync/service-core';
-import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules';
+import { DateTimeValue, ParameterLookup, RequestParameters } from '@powersync/service-sync-rules';
 import { expect, test, describe, beforeEach } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
+import { SqlBucketDescriptor } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js';
 
 export const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
 
@@ -34,18 +36,19 @@ const normalizeOplogData = (data: OplogEntry['data']) => {
  */
 export function registerDataStorageTests(generateStorageFactory: storage.TestStorageFactory) {
   test('save and load parameters', async () => {
-    const sync_rules = test_utils.testRules(`
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
-    data: [] 
-    `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    data: []
+    `
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -69,11 +72,12 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('t1')
       });
+
+      await batch.commit('1/1');
     });
 
-    const parameters = await bucketStorage.getParameterSets(result!.flushed_op, [
-      ParameterLookup.normalized('mybucket', '1', ['user1'])
-    ]);
+    const checkpoint = await bucketStorage.getCheckpoint();
+    const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]);
     expect(parameters).toEqual([
       {
         group_id: 'group1a'
@@ -82,20 +86,19 @@ bucket_definitions:
   });
 
   test('it should use the latest version', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE id = token_parameters.user_id
-    data: [] 
+    data: []
     `
-    );
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -105,8 +108,10 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('user1')
       });
+      await batch.commit('1/1');
     });
-    const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    const checkpoint1 = await bucketStorage.getCheckpoint();
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -116,11 +121,11 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('user1')
       });
+      await batch.commit('1/2');
     });
+    const checkpoint2 = await bucketStorage.getCheckpoint();
 
-    const parameters = await bucketStorage.getParameterSets(result2!.flushed_op, [
-      ParameterLookup.normalized('mybucket', '1', ['user1'])
-    ]);
+    const parameters = await checkpoint2.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]);
     expect(parameters).toEqual([
       {
         group_id: 'group2'
@@ -128,9 +133,7 @@ bucket_definitions:
     ]);
 
     // Use the checkpoint to get older data if relevant
-    const parameters2 = await bucketStorage.getParameterSets(result1!.flushed_op, [
-      ParameterLookup.normalized('mybucket', '1', ['user1'])
-    ]);
+    const parameters2 = await checkpoint1.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]);
     expect(parameters2).toEqual([
       {
         group_id: 'group1'
@@ -139,20 +142,19 @@ bucket_definitions:
   });
 
   test('it should use the latest version after updates', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT id AS todo_id
         FROM todos
         WHERE list_id IN token_parameters.list_id
-    data: [] 
+    data: []
     `
-    );
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const table = test_utils.makeTestTable('todos', ['id', 'list_id']);
 
@@ -176,9 +178,11 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('todo2')
       });
+
+      await batch.commit('1/1');
     });
 
-    const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       // Update the second todo item to now belong to list 1
       await batch.save({
         sourceTable: table,
@@ -189,12 +193,15 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('todo2')
       });
+
+      await batch.commit('1/1');
     });
 
     // We specifically request the todo_ids for both lists.
     // There removal operation for the association of `list2`::`todo2` should not interfere with the new
     // association of `list1`::`todo2`
-    const parameters = await bucketStorage.getParameterSets(result2!.flushed_op, [
+    const checkpoint = await bucketStorage.getCheckpoint();
+    const parameters = await checkpoint.getParameterSets([
       ParameterLookup.normalized('mybucket', '1', ['list1']),
       ParameterLookup.normalized('mybucket', '1', ['list2'])
     ]);
@@ -210,20 +217,19 @@ bucket_definitions:
   });
 
   test('save and load parameters with different number types', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE n1 = token_parameters.n1 and f2 = token_parameters.f2 and f3 = token_parameters.f3
     data: []
     `
-    );
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -236,21 +242,23 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('t1')
       });
+
+      await batch.commit('1/1');
     });
 
     const TEST_PARAMS = { group_id: 'group1' };
 
-    const checkpoint = result!.flushed_op;
+    const checkpoint = await bucketStorage.getCheckpoint();
 
-    const parameters1 = await bucketStorage.getParameterSets(checkpoint, [
+    const parameters1 = await checkpoint.getParameterSets([
       ParameterLookup.normalized('mybucket', '1', [314n, 314, 3.14])
     ]);
     expect(parameters1).toEqual([TEST_PARAMS]);
-    const parameters2 = await bucketStorage.getParameterSets(checkpoint, [
+    const parameters2 = await checkpoint.getParameterSets([
       ParameterLookup.normalized('mybucket', '1', [314, 314n, 3.14])
     ]);
     expect(parameters2).toEqual([TEST_PARAMS]);
-    const parameters3 = await bucketStorage.getParameterSets(checkpoint, [
+    const parameters3 = await checkpoint.getParameterSets([
       ParameterLookup.normalized('mybucket', '1', [314n, 314, 3])
     ]);
     expect(parameters3).toEqual([]);
@@ -261,20 +269,19 @@ bucket_definitions:
     // This specific case tested here cannot happen with postgres in practice, but we still
     // test this to ensure correct deserialization.
 
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE n1 = token_parameters.n1
     data: []
     `
-    );
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -298,31 +305,33 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('t1')
       });
+
+      await batch.commit('1/1');
     });
 
     const TEST_PARAMS = { group_id: 'group1' };
 
-    const checkpoint = result!.flushed_op;
+    const checkpoint = await bucketStorage.getCheckpoint();
 
-    const parameters1 = await bucketStorage.getParameterSets(checkpoint, [
+    const parameters1 = await checkpoint.getParameterSets([
       ParameterLookup.normalized('mybucket', '1', [1152921504606846976n])
     ]);
     expect(parameters1).toEqual([TEST_PARAMS]);
   });
 
   test('removing row', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `
-    );
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
 
       await batch.save({
@@ -339,9 +348,10 @@ bucket_definitions:
         tag: storage.SaveOperationTag.DELETE,
         beforeReplicaId: test_utils.rid('test1')
       });
+      await batch.commit('1/1');
     });
 
-    const checkpoint = result!.flushed_op;
+    const { checkpoint } = await bucketStorage.getCheckpoint();
 
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
     const data = batch[0].chunkData.data.map((d) => {
@@ -373,8 +383,9 @@ bucket_definitions:
   test('save and load parameters with workspaceId', async () => {
     const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', ['id']);
 
-    const sync_rules_content = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
     by_workspace:
       parameters:
@@ -382,13 +393,11 @@ bucket_definitions:
           workspace."userId" = token_parameters.user_id
       data: []
     `
-    );
-    const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules;
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules_content);
+    });
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules;
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: WORKSPACE_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -398,33 +407,38 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('workspace1')
       });
+      await batch.commit('1/1');
     });
-
-    const checkpoint = result!.flushed_op;
+    const checkpoint = await bucketStorage.getCheckpoint();
 
     const parameters = new RequestParameters({ sub: 'u1' }, {});
 
-    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
+    const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0];
 
     const lookups = q1.getLookups(parameters);
     expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]);
 
-    const parameter_sets = await bucketStorage.getParameterSets(checkpoint, lookups);
+    const parameter_sets = await checkpoint.getParameterSets(lookups);
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
-      getParameterSets(lookups) {
-        return bucketStorage.getParameterSets(checkpoint, lookups);
-      }
-    });
-    expect(buckets).toEqual([{ bucket: 'by_workspace["workspace1"]', priority: 3 }]);
+    const buckets = await sync_rules
+      .getBucketParameterQuerier(test_utils.querierOptions(parameters))
+      .querier.queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return checkpoint.getParameterSets(lookups);
+        }
+      });
+    expect(buckets).toEqual([
+      { bucket: 'by_workspace["workspace1"]', priority: 3, definition: 'by_workspace', inclusion_reasons: ['default'] }
+    ]);
   });
 
   test('save and load parameters with dynamic global buckets', async () => {
     const WORKSPACE_TABLE = test_utils.makeTestTable('workspace');
 
-    const sync_rules_content = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
     by_public_workspace:
       parameters:
@@ -432,13 +446,11 @@ bucket_definitions:
           workspace.visibility = 'public'
       data: []
     `
-    );
-    const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules;
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules_content);
+    });
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules;
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: WORKSPACE_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -468,38 +480,53 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('workspace3')
       });
+
+      await batch.commit('1/1');
     });
 
-    const checkpoint = result!.flushed_op;
+    const checkpoint = await bucketStorage.getCheckpoint();
 
     const parameters = new RequestParameters({ sub: 'unknown' }, {});
 
-    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
+    const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0];
 
     const lookups = q1.getLookups(parameters);
     expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]);
 
-    const parameter_sets = await bucketStorage.getParameterSets(checkpoint, lookups);
+    const parameter_sets = await checkpoint.getParameterSets(lookups);
     parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
 
-    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
-      getParameterSets(lookups) {
-        return bucketStorage.getParameterSets(checkpoint, lookups);
-      }
-    });
+    const buckets = await sync_rules
+      .getBucketParameterQuerier(test_utils.querierOptions(parameters))
+      .querier.queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return checkpoint.getParameterSets(lookups);
+        }
+      });
     buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
     expect(buckets).toEqual([
-      { bucket: 'by_public_workspace["workspace1"]', priority: 3 },
-      { bucket: 'by_public_workspace["workspace3"]', priority: 3 }
+      {
+        bucket: 'by_public_workspace["workspace1"]',
+        priority: 3,
+        definition: 'by_public_workspace',
+        inclusion_reasons: ['default']
+      },
+      {
+        bucket: 'by_public_workspace["workspace3"]',
+        priority: 3,
+        definition: 'by_public_workspace',
+        inclusion_reasons: ['default']
+      }
     ]);
   });
 
   test('multiple parameter queries', async () => {
     const WORKSPACE_TABLE = test_utils.makeTestTable('workspace');
 
-    const sync_rules_content = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
     by_workspace:
       parameters:
@@ -509,13 +536,11 @@ bucket_definitions:
             workspace.user_id = token_parameters.user_id
       data: []
     `
-    );
-    const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules;
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules_content);
+    });
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules;
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: WORKSPACE_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -557,56 +582,59 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('workspace4')
       });
+
+      await batch.commit('1/1');
     });
 
-    const checkpoint = result!.flushed_op;
+    const checkpoint = await bucketStorage.getCheckpoint();
 
     const parameters = new RequestParameters({ sub: 'u1' }, {});
 
     // Test intermediate values - could be moved to sync_rules.test.ts
-    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
+    const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0];
     const lookups1 = q1.getLookups(parameters);
     expect(lookups1).toEqual([ParameterLookup.normalized('by_workspace', '1', [])]);
 
-    const parameter_sets1 = await bucketStorage.getParameterSets(checkpoint, lookups1);
+    const parameter_sets1 = await checkpoint.getParameterSets(lookups1);
     parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const q2 = sync_rules.bucketDescriptors[0].parameterQueries[1];
+    const q2 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[1];
     const lookups2 = q2.getLookups(parameters);
     expect(lookups2).toEqual([ParameterLookup.normalized('by_workspace', '2', ['u1'])]);
 
-    const parameter_sets2 = await bucketStorage.getParameterSets(checkpoint, lookups2);
+    const parameter_sets2 = await checkpoint.getParameterSets(lookups2);
     parameter_sets2.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets2).toEqual([{ workspace_id: 'workspace3' }]);
 
     // Test final values - the important part
     const buckets = (
-      await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
-        getParameterSets(lookups) {
-          return bucketStorage.getParameterSets(checkpoint, lookups);
-        }
-      })
+      await sync_rules
+        .getBucketParameterQuerier(test_utils.querierOptions(parameters))
+        .querier.queryDynamicBucketDescriptions({
+          getParameterSets(lookups) {
+            return checkpoint.getParameterSets(lookups);
+          }
+        })
     ).map((e) => e.bucket);
     buckets.sort();
     expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']);
   });
 
   test('changing client ids', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data:
       - SELECT client_id as id, description FROM "%"
 `
-    );
-    await using factory = await generateStorageFactory();
-
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = TEST_TABLE;
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable,
         tag: storage.SaveOperationTag.INSERT,
@@ -638,8 +666,10 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('test2')
       });
+
+      await batch.commit('1/1');
     });
-    const checkpoint = result!.flushed_op;
+    const { checkpoint } = await bucketStorage.getCheckpoint();
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
     const data = batch[0].chunkData.data.map((d) => {
       return {
@@ -657,16 +687,16 @@ bucket_definitions:
   });
 
   test('re-apply delete', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `
-    );
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
@@ -690,9 +720,11 @@ bucket_definitions:
         tag: storage.SaveOperationTag.DELETE,
         beforeReplicaId: test_utils.rid('test1')
       });
+
+      await batch.commit('1/1');
     });
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
 
       await batch.save({
@@ -702,7 +734,7 @@ bucket_definitions:
       });
     });
 
-    const checkpoint = result!.flushed_op;
+    const { checkpoint } = await bucketStorage.getCheckpoint();
 
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
     const data = batch[0].chunkData.data.map((d) => {
@@ -732,16 +764,16 @@ bucket_definitions:
   });
 
   test('re-apply update + delete', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `
-    );
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
@@ -785,9 +817,11 @@ bucket_definitions:
         tag: storage.SaveOperationTag.DELETE,
         beforeReplicaId: test_utils.rid('test1')
       });
+
+      await batch.commit('1/1');
     });
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
 
       await batch.save({
@@ -815,9 +849,11 @@ bucket_definitions:
         tag: storage.SaveOperationTag.DELETE,
         beforeReplicaId: test_utils.rid('test1')
       });
+
+      await batch.commit('2/1');
     });
 
-    const checkpoint = result!.flushed_op;
+    const { checkpoint } = await bucketStorage.getCheckpoint();
 
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
 
@@ -850,18 +886,17 @@ bucket_definitions:
   });
 
   test('truncate parameters', async () => {
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
     data: []
     `
-    );
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
@@ -879,11 +914,9 @@ bucket_definitions:
       await batch.truncate([TEST_TABLE]);
     });
 
-    const { checkpoint } = await bucketStorage.getCheckpoint();
+    const checkpoint = await bucketStorage.getCheckpoint();
 
-    const parameters = await bucketStorage.getParameterSets(checkpoint, [
-      ParameterLookup.normalized('mybucket', '1', ['user1'])
-    ]);
+    const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]);
     expect(parameters).toEqual([]);
   });
 
@@ -896,16 +929,16 @@ bucket_definitions:
     // 1. Not getting the correct "current_data" state for each operation.
     // 2. Output order not being correct.
 
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
 `
-    );
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     // Pre-setup
     const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -1048,14 +1081,6 @@ bucket_definitions:
   });
 
   test('changed data with replica identity full', async () => {
-    const sync_rules = test_utils.testRules(
-      `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test"
-`
-    );
     function rid2(id: string, description: string) {
       return getUuidReplicaIdentityBson({ id, description }, [
         { name: 'id', type: 'VARCHAR', typeId: 25 },
@@ -1063,7 +1088,15 @@ bucket_definitions:
       ]);
     }
     await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    const syncRules = await factory.updateSyncRules({
+      content: `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
 
@@ -1155,14 +1188,6 @@ bucket_definitions:
   });
 
   test('unchanged data with replica identity full', async () => {
-    const sync_rules = test_utils.testRules(
-      `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test"
-`
-    );
     function rid2(id: string, description: string) {
       return getUuidReplicaIdentityBson({ id, description }, [
         { name: 'id', type: 'VARCHAR', typeId: 25 },
@@ -1171,7 +1196,15 @@ bucket_definitions:
     }
 
     await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    const syncRules = await factory.updateSyncRules({
+      content: `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
 
@@ -1260,18 +1293,18 @@ bucket_definitions:
     // but large enough in size to be split over multiple returned batches.
     // The specific batch splits is an implementation detail of the storage driver,
     // and the test will have to updated when other implementations are added.
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `
-    );
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
 
       const largeDescription = '0123456789'.repeat(12_000_00);
@@ -1316,9 +1349,11 @@ bucket_definitions:
         },
         afterReplicaId: test_utils.rid('test3')
       });
+
+      await batch.commit('1/1');
     });
 
-    const checkpoint = result!.flushed_op;
+    const { checkpoint } = await bucketStorage.getCheckpoint();
 
     const options: storage.BucketDataBatchOptions = {
       chunkLimitBytes: 16 * 1024 * 1024
@@ -1367,18 +1402,18 @@ bucket_definitions:
 
   test('long batch', async () => {
     // Test syncing a batch of data that is limited by count.
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `
-    );
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
 
       for (let i = 1; i <= 6; i++) {
@@ -1392,9 +1427,11 @@ bucket_definitions:
           afterReplicaId: `test${i}`
         });
       }
+
+      await batch.commit('1/1');
     });
 
-    const checkpoint = result!.flushed_op;
+    const { checkpoint } = await bucketStorage.getCheckpoint();
 
     const batch1 = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), { limit: 4 })
@@ -1441,8 +1478,9 @@ bucket_definitions:
 
   describe('batch has_more', () => {
     const setup = async (options: BucketDataBatchOptions) => {
-      const sync_rules = test_utils.testRules(
-        `
+      await using factory = await generateStorageFactory();
+      const syncRules = await factory.updateSyncRules({
+        content: `
   bucket_definitions:
     global1:
       data:
@@ -1451,11 +1489,10 @@ bucket_definitions:
       data:
         - SELECT id, description FROM test WHERE bucket = 'global2'
   `
-      );
-      await using factory = await generateStorageFactory();
-      const bucketStorage = factory.getInstance(sync_rules);
+      });
+      const bucketStorage = factory.getInstance(syncRules);
 
-      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
         const sourceTable = TEST_TABLE;
 
         for (let i = 1; i <= 10; i++) {
@@ -1470,9 +1507,11 @@ bucket_definitions:
             afterReplicaId: `test${i}`
           });
         }
+
+        await batch.commit('1/1');
       });
 
-      const checkpoint = result!.flushed_op;
+      const { checkpoint } = await bucketStorage.getCheckpoint();
       return await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
@@ -1603,15 +1642,18 @@ bucket_definitions:
 
     const r = await f.configureSyncRules({ content: 'bucket_definitions: {}', validate: false });
     const storage = f.getInstance(r.persisted_sync_rules!);
-    await storage.autoActivate();
+    await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('1/0');
+    });
 
     const metrics2 = await f.getStorageMetrics();
     expect(metrics2).toMatchSnapshot();
   });
 
   test('invalidate cached parsed sync rules', async () => {
-    const sync_rules_content = test_utils.testRules(
-      `
+    await using bucketStorageFactory = await generateStorageFactory();
+    const syncRules = await bucketStorageFactory.updateSyncRules({
+      content: `
 bucket_definitions:
     by_workspace:
       parameters:
@@ -1619,10 +1661,8 @@ bucket_definitions:
           workspace."userId" = token_parameters.user_id
       data: []
     `
-    );
-
-    await using bucketStorageFactory = await generateStorageFactory();
-    const syncBucketStorage = bucketStorageFactory.getInstance(sync_rules_content);
+    });
+    const syncBucketStorage = bucketStorageFactory.getInstance(syncRules);
 
     const parsedSchema1 = syncBucketStorage.getParsedSyncRules({
       defaultSchema: 'public'
@@ -1651,12 +1691,11 @@ bucket_definitions:
       content: `
 bucket_definitions:
   mybucket:
-    data: [] 
+    data: []
     `,
       validate: false
     });
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    await bucketStorage.autoActivate();
 
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
@@ -1697,7 +1736,6 @@ bucket_definitions:
       validate: false
     });
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    await bucketStorage.autoActivate();
 
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
@@ -1760,7 +1798,6 @@ bucket_definitions:
       validate: false
     });
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    await bucketStorage.autoActivate();
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
     const abortController = new AbortController();
@@ -1801,7 +1838,6 @@ bucket_definitions:
       validate: false
     });
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    await bucketStorage.autoActivate();
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
     const abortController = new AbortController();
@@ -1845,7 +1881,6 @@ bucket_definitions:
       validate: false
     });
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    await bucketStorage.autoActivate();
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
     const abortController = new AbortController();
@@ -1916,17 +1951,17 @@ bucket_definitions:
     // Test syncing a batch of data that is small in count,
     // but large enough in size to be split over multiple returned chunks.
     // Similar to the above test, but splits over 1MB chunks.
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules({
+      content: `
   bucket_definitions:
     global:
       data:
         - SELECT id FROM test
         - SELECT id FROM test_ignore WHERE false
   `
-    );
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
+    });
+    const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = test_utils.makeTestTable('test', ['id']);
     const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id']);
@@ -1961,4 +1996,70 @@ bucket_definitions:
     // we expect 0n and 1n, or 1n and 2n.
     expect(checkpoint2).toBeGreaterThan(checkpoint1);
   });
+
+  test('data with custom types', async () => {
+    await using factory = await generateStorageFactory();
+    const testValue = {
+      sourceTable: TEST_TABLE,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 't1',
+        description: new DateTimeValue('2025-08-28T11:30:00')
+      },
+      afterReplicaId: test_utils.rid('t1')
+    } satisfies SaveOptions;
+
+    {
+      // First, deploy old sync rules and row with date time value
+      const syncRules = await factory.updateSyncRules({
+        content: `
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id, description FROM test
+  `
+      });
+      const bucketStorage = factory.getInstance(syncRules);
+      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        await batch.save(testValue);
+        await batch.commit('1/1');
+      });
+
+      const { checkpoint } = await bucketStorage.getCheckpoint();
+      const batch = await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]))
+      );
+      expect(batch[0].chunkData.data).toMatchObject([
+        {
+          data: '{"id":"t1","description":"2025-08-28 11:30:00"}'
+        }
+      ]);
+    }
+
+    const syncRules = await factory.updateSyncRules({
+      content: `
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id, description FROM test
+  
+  config:
+    edition: 2
+  `
+    });
+    const bucketStorage = factory.getInstance(syncRules);
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save(testValue);
+      await batch.commit('1/2');
+    });
+    const { checkpoint } = await bucketStorage.getCheckpoint();
+    const batch = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['2#global[]', 0n]]))
+    );
+    expect(batch[0].chunkData.data).toMatchObject([
+      {
+        data: '{"id":"t1","description":"2025-08-28T11:30:00"}'
+      }
+    ]);
+  });
 }