@powersync/service-core-tests 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/dist/tests/register-data-storage-data-tests.js

@@ -27,13 +27,13 @@ export function registerDataStorageDataTests(config) {
     const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
     test('removing row', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `, { storageVersion }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -52,9 +52,9 @@ bucket_definitions:
             beforeReplicaId: test_utils.rid('test1')
         });
         await writer.commit('1/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -79,13 +79,13 @@ bucket_definitions:
     });
     test('insert after delete in new batch', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `, { storageVersion }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -105,9 +105,9 @@ bucket_definitions:
             afterReplicaId: test_utils.rid('test1')
         });
         await writer.commit('2/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -129,13 +129,13 @@ bucket_definitions:
     test('update after delete in new batch', async () => {
         // Update after delete may not be common, but the storage layer should handle it in an eventually-consistent way.
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `, { storageVersion }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -159,9 +159,9 @@ bucket_definitions:
             afterReplicaId: test_utils.rid('test1')
         });
         await writer.commit('2/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -182,7 +182,7 @@ bucket_definitions:
     });
     test('insert after delete in same batch', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -190,7 +190,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -209,9 +209,9 @@ bucket_definitions:
             afterReplicaId: test_utils.rid('test1')
         });
         await writer.commit('1/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -232,7 +232,7 @@ bucket_definitions:
     });
     test('(insert, delete, insert), (delete)', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -240,7 +240,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         {
             await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
             const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -281,9 +281,9 @@ bucket_definitions:
             });
             await writer.commit('2/1');
         }
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
         expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
         const data = batch[0].chunkData.data.map((d) => {
             return {
@@ -296,7 +296,7 @@ bucket_definitions:
     });
     test('changing client ids', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -304,7 +304,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -339,8 +339,8 @@ bucket_definitions:
             afterReplicaId: test_utils.rid('test2')
         });
         await writer.commit('1/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
+        const checkpoint = await bucketStorage.getCheckpoint();
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -356,7 +356,7 @@ bucket_definitions:
     });
     test('re-apply delete', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -364,7 +364,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -390,9 +390,9 @@ bucket_definitions:
             beforeReplicaId: test_utils.rid('test1')
         });
         await writer.flush();
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -417,13 +417,13 @@ bucket_definitions:
     });
     test('re-apply update + delete', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `, { storageVersion }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -487,9 +487,9 @@ bucket_definitions:
             beforeReplicaId: test_utils.rid('test1')
         });
         await writer.commit('2/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -523,13 +523,13 @@ bucket_definitions:
         // 1. Not getting the correct "current_data" state for each operation.
         // 2. Output order not being correct.
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
 `, { storageVersion }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         // Pre-setup
@@ -627,7 +627,7 @@ bucket_definitions:
         const result2 = await writer.flush();
         const checkpoint2 = result2.flushed_op;
         const request = bucketRequest(syncRules, 'global[]', checkpoint1);
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -659,7 +659,7 @@ bucket_definitions:
             ]);
         }
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -667,7 +667,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config);
         // Pre-setup
@@ -713,7 +713,7 @@ bucket_definitions:
         const result3 = await writer.flush();
         const checkpoint3 = result3.flushed_op;
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -755,7 +755,7 @@ bucket_definitions:
             ]);
         }
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -763,7 +763,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config);
         // Pre-setup
@@ -809,7 +809,7 @@ bucket_definitions:
         const result3 = await writer.flush();
         const checkpoint3 = result3.flushed_op;
         const request = bucketRequest(syncRules, 'global[]');
-        const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
+        const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
         const data = batch[0].chunkData.data.map((d) => {
             return {
                 op: d.op,
@@ -842,7 +842,7 @@ bucket_definitions:
         // The specific batch splits is an implementation detail of the storage driver,
         // and the test will have to updated when other implementations are added.
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -850,7 +850,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -893,12 +893,12 @@ bucket_definitions:
             afterReplicaId: test_utils.rid('test3')
         });
         await writer.commit('1/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const options = {
             chunkLimitBytes: 16 * 1024 * 1024
         };
         const request = bucketRequest(syncRules, 'global[]');
-        const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
+        const batch1 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
         expect(test_utils.getBatchData(batch1)).toEqual([
             { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
             { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
@@ -908,7 +908,7 @@ bucket_definitions:
             has_more: true,
             next_after: '2'
         });
-        const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
+        const batch2 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
         expect(test_utils.getBatchData(batch2)).toEqual([
             { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
             { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
@@ -918,14 +918,15 @@ bucket_definitions:
             has_more: false,
             next_after: '4'
         });
-        const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
+        const batch3 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
         expect(test_utils.getBatchData(batch3)).toEqual([]);
         expect(test_utils.getBatchMeta(batch3)).toEqual(null);
     });
     test('long batch', async () => {
-        // Test syncing a batch of data that is limited by count.
+        // Test syncing a batch of data that is limited by count. Compressed storage limits persisted
+        // documents and finishes yielding all operations from a document once it has been hydrated.
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -933,7 +934,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -949,42 +950,57 @@ bucket_definitions:
             });
         }
         await writer.commit('1/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
-        const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
-        expect(test_utils.getBatchData(batch1)).toEqual([
+        const batch1 = await test_utils.getSingleBatchItem(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
+        const allOperations = [
             { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
             { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
             { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
-            { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
-        ]);
-        expect(test_utils.getBatchMeta(batch1)).toEqual({
-            after: '0',
-            has_more: true,
-            next_after: '4'
-        });
-        const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
-            limit: 4
-        }));
-        expect(test_utils.getBatchData(batch2)).toEqual([
+            { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
             { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
             { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
-        ]);
-        expect(test_utils.getBatchMeta(batch2)).toEqual({
-            after: '4',
-            has_more: false,
-            next_after: '6'
-        });
-        const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
-            limit: 4
-        }));
-        expect(test_utils.getBatchData(batch3)).toEqual([]);
-        expect(test_utils.getBatchMeta(batch3)).toEqual(null);
+        ];
+        if (config.compressedBucketStorage) {
+            expect(test_utils.getBatchData(batch1)).toEqual(allOperations);
+            expect(test_utils.getBatchMeta(batch1)).toEqual({
+                after: '0',
+                has_more: false,
+                next_after: '6'
+            });
+            const batch2 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
+                limit: 4
+            }));
+            expect(test_utils.getBatchData(batch2)).toEqual([]);
+            expect(test_utils.getBatchMeta(batch2)).toEqual(null);
+        }
+        else {
+            expect(test_utils.getBatchData(batch1)).toEqual(allOperations.slice(0, 4));
+            expect(test_utils.getBatchMeta(batch1)).toEqual({
+                after: '0',
+                has_more: true,
+                next_after: '4'
+            });
+            const batch2 = await test_utils.getSingleBatchItem(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
+                limit: 4
+            }));
+            expect(test_utils.getBatchData(batch2)).toEqual(allOperations.slice(4));
+            expect(test_utils.getBatchMeta(batch2)).toEqual({
+                after: '4',
+                has_more: false,
+                next_after: '6'
+            });
+            const batch3 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
+                limit: 4
+            }));
+            expect(test_utils.getBatchData(batch3)).toEqual([]);
+            expect(test_utils.getBatchMeta(batch3)).toEqual(null);
+        }
     });
     describe('batch has_more', () => {
-        const setup = async (options) => {
+        const setup = async (options, commitAfter) => {
             await using factory = await generateStorageFactory();
-            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+            const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
   bucket_definitions:
     global1:
       data:
@@ -993,7 +1009,7 @@ bucket_definitions:
       data:
         - SELECT id, description FROM test WHERE bucket = 'global2'
   `, { storageVersion }));
-            const bucketStorage = factory.getInstance(syncRules);
+            const bucketStorage = factory.getInstance(replicationStream);
             await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
             const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
             await writer.markAllSnapshotDone('1/1');
@@ -1008,28 +1024,39 @@ bucket_definitions:
                     },
                     afterReplicaId: `test${i}`
                 });
+                if (i == commitAfter) {
+                    await writer.commit('1/1');
+                }
             }
-            await writer.commit('1/1');
-            const { checkpoint } = await bucketStorage.getCheckpoint();
+            await writer.commit(commitAfter == null ? '1/1' : '1/2');
+            const checkpoint = await bucketStorage.getCheckpoint();
             const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
             const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
-            const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options));
+            const batch = await new test_utils.StorageDataHelpers(bucketStorage, syncRules).getAllBucketData([global1Request, global2Request], checkpoint, options);
             return { batch, global1Request, global2Request };
         };
         test('batch has_more (1)', async () => {
-            const { batch, global1Request, global2Request } = await setup({ limit: 5 });
-            expect(batch.length).toEqual(2);
+            // The commit boundary creates two compressed documents for global2. The limit selects the global1
+            // document and the first global2 document, leaving the second global2 document for the next batch.
+            const { batch, global1Request, global2Request } = await setup({ limit: 2 }, 5);
+            expect(batch.length).toBeGreaterThanOrEqual(2);
             expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
             expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
             expect(test_utils.getBatchData(batch[0])).toEqual([
                 { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
             ]);
-            expect(test_utils.getBatchData(batch[1])).toEqual([
+            const global2Operations = [
                 { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
                 { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
                 { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
-                { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }
-            ]);
+                { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
+                { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
+                { op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
+                { op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
+                { op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
+                { op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
+            ];
+            expect(test_utils.getBatchData(batch[1])).toEqual(config.compressedBucketStorage ? global2Operations.slice(0, 4) : global2Operations.slice(0, 1));
             expect(test_utils.getBatchMeta(batch[0])).toEqual({
                 after: '0',
                 has_more: false,
@@ -1038,7 +1065,7 @@ bucket_definitions:
             expect(test_utils.getBatchMeta(batch[1])).toEqual({
                 after: '0',
                 has_more: true,
-                next_after: '5'
+                next_after: config.compressedBucketStorage ? '5' : '2'
             });
         });
         test('batch has_more (2)', async () => {
@@ -1074,43 +1101,77 @@ bucket_definitions:
         test('batch has_more (3)', async () => {
             // 50 bytes is more than 1 row, less than 2 rows
             const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
-            expect(batch.length).toEqual(3);
-            expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
-            expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
-            expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
-            expect(test_utils.getBatchData(batch[0])).toEqual([
-                { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
-            ]);
-            expect(test_utils.getBatchData(batch[1])).toEqual([
-                { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 }
-            ]);
-            expect(test_utils.getBatchData(batch[2])).toEqual([
-                { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
-            ]);
-            expect(test_utils.getBatchMeta(batch[0])).toEqual({
-                after: '0',
-                has_more: false,
-                next_after: '1'
-            });
-            expect(test_utils.getBatchMeta(batch[1])).toEqual({
-                after: '0',
-                has_more: true,
-                next_after: '2'
-            });
-            expect(test_utils.getBatchMeta(batch[2])).toEqual({
-                after: '2',
-                has_more: true,
-                next_after: '3'
-            });
+            if (config.compressedBucketStorage) {
+                // In v3+, ops in the same bucket share a document. Once hydrated, the entire document is yielded even
+                // when it exceeds the byte limit; the byte limit controls whether another document is fetched.
+                expect(batch.length).toEqual(2);
+                expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
+                expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
+                expect(test_utils.getBatchData(batch[0])).toEqual([
+                    { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+                ]);
+                expect(test_utils.getBatchData(batch[1])).toEqual([
+                    { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
+                    { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
+                    { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
+                    { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
+                    { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
+                    { op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
+                    { op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
+                    { op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
+                    { op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
+                ]);
+                expect(test_utils.getBatchMeta(batch[0])).toEqual({
+                    after: '0',
+                    has_more: false,
+                    next_after: '1'
+                });
+                expect(test_utils.getBatchMeta(batch[1])).toEqual({
+                    after: '0',
+                    has_more: false,
+                    next_after: '10'
+                });
+            }
+            else {
+                expect(batch.length).toBeGreaterThanOrEqual(3);
+                expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
+                expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
+                expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
+                expect(test_utils.getBatchData(batch[0])).toEqual([
+                    { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+                ]);
+                expect(test_utils.getBatchData(batch[1])).toEqual([
+                    { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 }
+                ]);
+                expect(test_utils.getBatchData(batch[2])).toEqual([
+                    { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+                ]);
+                expect(test_utils.getBatchMeta(batch[0])).toEqual({
+                    after: '0',
+                    has_more: false,
+                    next_after: '1'
+                });
+                expect(test_utils.getBatchMeta(batch[1])).toEqual({
+                    after: '0',
+                    has_more: true,
+                    next_after: '2'
+                });
+                expect(test_utils.getBatchMeta(batch[2])).toEqual({
+                    after: '2',
+                    has_more: true,
+                    next_after: '3'
+                });
+            }
         });
     });
     test('empty storage metrics', async () => {
         await using f = await generateStorageFactory({ dropAll: true });
         const metrics = await f.getStorageMetrics();
-        expect(metrics).toEqual({
+        expect(metrics).toMatchObject({
             operations_size_bytes: 0,
             parameters_size_bytes: 0,
-            replication_size_bytes: 0
+            replication_size_bytes: 0,
+            object_storage_size_bytes: 0
         });
         const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
         const storage = f.getInstance(r.persisted_sync_rules);
@@ -1125,7 +1186,7 @@ bucket_definitions:
         // but large enough in size to be split over multiple returned chunks.
         // Similar to the above test, but splits over 1MB chunks.
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
   bucket_definitions:
     global:
       data:
@@ -1134,7 +1195,7 @@ bucket_definitions:
   `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
         const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
@@ -1166,7 +1227,7 @@ bucket_definitions:
     });
     test('unchanged checksums', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -1174,7 +1235,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -1188,23 +1249,25 @@ bucket_definitions:
             afterReplicaId: test_utils.rid('test1')
         });
         await writer.commit('1/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         const request = bucketRequest(syncRules, 'global[]');
         const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
         expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
-        const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
+        const checksums2 = [
+            ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint.checkpoint + 1n), [request])).values()
+        ];
         expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
     });
     testChecksumBatching(config);
     test('empty checkpoints (1)', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
 `, { storageVersion }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         await writer.markAllSnapshotDone('1/1');
         await writer.commit('1/1');
@@ -1224,7 +1287,7 @@ bucket_definitions:
     });
     test('empty checkpoints (2)', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -1232,7 +1295,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
@@ -1263,7 +1326,7 @@ bucket_definitions:
     });
     test('empty checkpoints (sync rule activation)', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -1271,7 +1334,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
         expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
@@ -1290,8 +1353,8 @@ bucket_definitions:
         // Now, the checkpoint should advance the sync rules active.
         const cp2 = await bucketStorage.getCheckpoint();
         expect(cp2.lsn).toEqual('3/1');
-        const activeSyncRules = await factory.getActiveSyncRulesContent();
-        expect(activeSyncRules?.id).toEqual(syncRules.id);
+        const activeSyncRules = await factory.getActiveSyncConfig();
+        expect(activeSyncRules?.content.replicationStreamId).toEqual(syncRules.replicationStreamId);
         // At this point, it should be a truely empty checkpoint
         const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
         expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
@@ -1301,7 +1364,7 @@ bucket_definitions:
     });
     test.runIf(storageVersion >= 3)('deleting while streaming', async () => {
         await using factory = await generateStorageFactory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
@@ -1309,7 +1372,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using snapshotWriter = await bucketStorage.createWriter({
             ...test_utils.BATCH_OPTIONS,
             skipExistingRows: true
@@ -1344,7 +1407,7 @@ bucket_definitions:
         await streamingWriter.keepalive('3/1');
         const cp = await bucketStorage.getCheckpoint();
         expect(cp.lsn).toEqual('3/1');
-        const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]));
+        const data = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(cp, [bucketRequest(syncRules, 'global[]')]));
         expect(data).toEqual([]);
     });
 }
@@ -1357,7 +1420,7 @@ export function testChecksumBatching(config) {
     const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
     test('checksums for multiple buckets', async () => {
         await using factory = await config.factory();
-        const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+        const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
 bucket_definitions:
   user:
     parameters: select request.user_id() as user_id
@@ -1366,7 +1429,7 @@ bucket_definitions:
 `, {
             storageVersion
         }));
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = factory.getInstance(replicationStream);
         await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
         await writer.markAllSnapshotDone('1/1');
@@ -1386,7 +1449,7 @@ bucket_definitions:
             }
         }
         await writer.commit('1/1');
-        const { checkpoint } = await bucketStorage.getCheckpoint();
+        const checkpoint = await bucketStorage.getCheckpoint();
         bucketStorage.clearChecksumCache();
         const users = ['u1', 'u2', 'u3', 'u4'];
         const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
@@ -1411,7 +1474,11 @@ streams:
   invalid:
     query: UPDATE test SET foo = 'bar' RETURNING *
 `));
-        const { errors } = deployed.parsed({ defaultSchema: 'ignored' }).syncConfigWithErrors;
+        const [deployedContent] = deployed.syncConfigContent;
+        expect(deployedContent).toBeDefined();
+        const [deployedConfig] = deployedContent.parsed({ defaultSchema: 'ignored' }).syncConfigs;
+        expect(deployedConfig).toBeDefined();
+        const { errors } = deployedConfig;
         expect(errors).toHaveLength(1);
         expect(errors[0].message).toStrictEqual('Expected a SELECT statement');
         expect(errors[0].location).toStrictEqual({