@powersync/service-core-tests 0.14.0 → 0.15.0

package/dist/tests/register-data-storage-data-tests.js

@@ -50,10 +50,11 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
     var e = new Error(message);
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 });
-import { getUuidReplicaIdentityBson, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { CURRENT_STORAGE_VERSION, getUuidReplicaIdentityBson, reduceBucket, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { describe, expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
-import { bucketRequest, bucketRequestMap, bucketRequests, TEST_TABLE } from './util.js';
+import { bucketRequest } from '../test-utils/test-utils-index.js';
+import { bucketRequestMap, bucketRequests } from './util.js';
 /**
  * Normalize data from OplogEntries for comparison in tests.
  * Tests typically expect the stringified result
@@ -74,7 +75,10 @@ const normalizeOplogData = (data) => {
  *
  * ```
  */
-export function registerDataStorageDataTests(generateStorageFactory) {
+export function registerDataStorageDataTests(config) {
+    const generateStorageFactory = config.factory;
+    const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+    const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
     test('removing row', async () => {
         const env_1 = { stack: [], error: void 0, hasError: false };
         try {
@@ -84,10 +88,11 @@ bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`));
+`, { storageVersion }));
             const bucketStorage = factory.getInstance(syncRules);
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
                 const sourceTable = TEST_TABLE;
+                await batch.markAllSnapshotDone('1/1');
                 await batch.save({
                     sourceTable,
                     tag: storage.SaveOperationTag.INSERT,
@@ -124,7 +129,7 @@ bucket_definitions:
             ];
             expect(checksums).toEqual([
                 {
-                    bucket: bucketRequest(syncRules, 'global[]'),
+                    bucket: bucketRequest(syncRules, 'global[]').bucket,
                     checksum: (c1 + c2) & 0xffffffff,
                     count: 2
                 }
@@ -140,19 +145,296 @@ bucket_definitions:
                 await result_1;
         }
     });
-    test('changing client ids', async () => {
+    test('insert after delete in new batch', async () => {
         const env_2 = { stack: [], error: void 0, hasError: false };
         try {
             const factory = __addDisposableResource(env_2, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, { storageVersion }));
+            const bucketStorage = factory.getInstance(syncRules);
+            const sourceTable = TEST_TABLE;
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.DELETE,
+                    beforeReplicaId: test_utils.rid('test1')
+                });
+                await batch.commit('0/1');
+            });
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                const sourceTable = TEST_TABLE;
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.INSERT,
+                    after: {
+                        id: 'test1',
+                        description: 'test1'
+                    },
+                    afterReplicaId: test_utils.rid('test1')
+                });
+                await batch.commit('2/1');
+            });
+            const { checkpoint } = await bucketStorage.getCheckpoint();
+            const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
+            const data = batch[0].chunkData.data.map((d) => {
+                return {
+                    op: d.op,
+                    object_id: d.object_id,
+                    checksum: d.checksum
+                };
+            });
+            const c1 = 2871785649;
+            expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
+            const checksums = [
+                ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
+            ];
+            expect(checksums).toEqual([
+                {
+                    bucket: bucketRequest(syncRules, 'global[]').bucket,
+                    checksum: c1 & 0xffffffff,
+                    count: 1
+                }
+            ]);
+        }
+        catch (e_2) {
+            env_2.error = e_2;
+            env_2.hasError = true;
+        }
+        finally {
+            const result_2 = __disposeResources(env_2);
+            if (result_2)
+                await result_2;
+        }
+    });
+    test('update after delete in new batch', async () => {
+        const env_3 = { stack: [], error: void 0, hasError: false };
+        try {
+            // Update after delete may not be common, but the storage layer should handle it in an eventually-consistent way.
+            const factory = __addDisposableResource(env_3, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, { storageVersion }));
+            const bucketStorage = factory.getInstance(syncRules);
+            const sourceTable = TEST_TABLE;
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.DELETE,
+                    beforeReplicaId: test_utils.rid('test1')
+                });
+                await batch.commit('0/1');
+            });
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                const sourceTable = TEST_TABLE;
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.UPDATE,
+                    before: {
+                        id: 'test1'
+                    },
+                    after: {
+                        id: 'test1',
+                        description: 'test1'
+                    },
+                    beforeReplicaId: test_utils.rid('test1'),
+                    afterReplicaId: test_utils.rid('test1')
+                });
+                await batch.commit('2/1');
+            });
+            const { checkpoint } = await bucketStorage.getCheckpoint();
+            const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
+            const data = batch[0].chunkData.data.map((d) => {
+                return {
+                    op: d.op,
+                    object_id: d.object_id,
+                    checksum: d.checksum
+                };
+            });
+            const c1 = 2871785649;
+            expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
+            const checksums = [
+                ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
+            ];
+            expect(checksums).toEqual([
+                {
+                    bucket: bucketRequest(syncRules, 'global[]').bucket,
+                    checksum: c1 & 0xffffffff,
+                    count: 1
+                }
+            ]);
+        }
+        catch (e_3) {
+            env_3.error = e_3;
+            env_3.hasError = true;
+        }
+        finally {
+            const result_3 = __disposeResources(env_3);
+            if (result_3)
+                await result_3;
+        }
+    });
+    test('insert after delete in same batch', async () => {
+        const env_4 = { stack: [], error: void 0, hasError: false };
+        try {
+            const factory = __addDisposableResource(env_4, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, {
+                storageVersion
+            }));
+            const bucketStorage = factory.getInstance(syncRules);
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                const sourceTable = TEST_TABLE;
+                await batch.markAllSnapshotDone('1/1');
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.DELETE,
+                    beforeReplicaId: test_utils.rid('test1')
+                });
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.INSERT,
+                    after: {
+                        id: 'test1',
+                        description: 'test1'
+                    },
+                    afterReplicaId: test_utils.rid('test1')
+                });
+                await batch.commit('1/1');
+            });
+            const { checkpoint } = await bucketStorage.getCheckpoint();
+            const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
+            const data = batch[0].chunkData.data.map((d) => {
+                return {
+                    op: d.op,
+                    object_id: d.object_id,
+                    checksum: d.checksum
+                };
+            });
+            const c1 = 2871785649;
+            expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
+            const checksums = [
+                ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
+            ];
+            expect(checksums).toEqual([
+                {
+                    bucket: bucketRequest(syncRules, 'global[]').bucket,
+                    checksum: c1 & 0xffffffff,
+                    count: 1
+                }
+            ]);
+        }
+        catch (e_4) {
+            env_4.error = e_4;
+            env_4.hasError = true;
+        }
+        finally {
+            const result_4 = __disposeResources(env_4);
+            if (result_4)
+                await result_4;
+        }
+    });
+    test('(insert, delete, insert), (delete)', async () => {
+        const env_5 = { stack: [], error: void 0, hasError: false };
+        try {
+            const factory = __addDisposableResource(env_5, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, {
+                storageVersion
+            }));
+            const bucketStorage = factory.getInstance(syncRules);
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                const sourceTable = TEST_TABLE;
+                await batch.markAllSnapshotDone('1/1');
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.INSERT,
+                    after: {
+                        id: 'test1',
+                        description: 'test1'
+                    },
+                    afterReplicaId: test_utils.rid('test1')
+                });
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.DELETE,
+                    beforeReplicaId: test_utils.rid('test1')
+                });
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.INSERT,
+                    after: {
+                        id: 'test1',
+                        description: 'test1'
+                    },
+                    afterReplicaId: test_utils.rid('test1')
+                });
+                await batch.commit('1/1');
+            });
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                const sourceTable = TEST_TABLE;
+                await batch.markAllSnapshotDone('1/1');
+                await batch.save({
+                    sourceTable,
+                    tag: storage.SaveOperationTag.DELETE,
+                    beforeReplicaId: test_utils.rid('test1')
+                });
+                await batch.commit('2/1');
+            });
+            const { checkpoint } = await bucketStorage.getCheckpoint();
+            const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
+            expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
+            const data = batch[0].chunkData.data.map((d) => {
+                return {
+                    op: d.op,
+                    object_id: d.object_id,
+                    checksum: d.checksum
+                };
+            });
+            expect(data).toMatchSnapshot();
+        }
+        catch (e_5) {
+            env_5.error = e_5;
+            env_5.hasError = true;
+        }
+        finally {
+            const result_5 = __disposeResources(env_5);
+            if (result_5)
+                await result_5;
+        }
+    });
+    test('changing client ids', async () => {
+        const env_6 = { stack: [], error: void 0, hasError: false };
+        try {
+            const factory = __addDisposableResource(env_6, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT client_id as id, description FROM "%"
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
             const sourceTable = TEST_TABLE;
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 await batch.save({
                     sourceTable,
                     tag: storage.SaveOperationTag.INSERT,
@@ -200,29 +482,32 @@ bucket_definitions:
                 { op: 'PUT', object_id: 'client2' }
             ]);
         }
-        catch (e_2) {
-            env_2.error = e_2;
-            env_2.hasError = true;
+        catch (e_6) {
+            env_6.error = e_6;
+            env_6.hasError = true;
         }
         finally {
-            const result_2 = __disposeResources(env_2);
-            if (result_2)
-                await result_2;
+            const result_6 = __disposeResources(env_6);
+            if (result_6)
+                await result_6;
         }
     });
     test('re-apply delete', async () => {
-        const env_3 = { stack: [], error: void 0, hasError: false };
+        const env_7 = { stack: [], error: void 0, hasError: false };
         try {
-            const factory = __addDisposableResource(env_3, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_7, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
                 const sourceTable = TEST_TABLE;
+                await batch.markAllSnapshotDone('1/1');
                 await batch.save({
                     sourceTable,
                     tag: storage.SaveOperationTag.INSERT,
@@ -270,34 +555,35 @@ bucket_definitions:
             ];
             expect(checksums).toEqual([
                 {
-                    bucket: bucketRequest(syncRules, 'global[]'),
+                    bucket: bucketRequest(syncRules, 'global[]').bucket,
                     checksum: (c1 + c2) & 0xffffffff,
                     count: 2
                 }
             ]);
         }
-        catch (e_3) {
-            env_3.error = e_3;
-            env_3.hasError = true;
+        catch (e_7) {
+            env_7.error = e_7;
+            env_7.hasError = true;
         }
         finally {
-            const result_3 = __disposeResources(env_3);
-            if (result_3)
-                await result_3;
+            const result_7 = __disposeResources(env_7);
+            if (result_7)
+                await result_7;
         }
     });
     test('re-apply update + delete', async () => {
-        const env_4 = { stack: [], error: void 0, hasError: false };
+        const env_8 = { stack: [], error: void 0, hasError: false };
         try {
-            const factory = __addDisposableResource(env_4, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`));
+`, { storageVersion }));
             const bucketStorage = factory.getInstance(syncRules);
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 const sourceTable = TEST_TABLE;
                 await batch.save({
                     sourceTable,
@@ -310,6 +596,7 @@ bucket_definitions:
                 });
             });
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 const sourceTable = TEST_TABLE;
                 await batch.save({
                     sourceTable,
@@ -337,6 +624,7 @@ bucket_definitions:
                 await batch.commit('1/1');
             });
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 const sourceTable = TEST_TABLE;
                 await batch.save({
                     sourceTable,
@@ -385,24 +673,24 @@ bucket_definitions:
             ];
             expect(checksums).toEqual([
                 {
-                    bucket: bucketRequest(syncRules, 'global[]'),
+                    bucket: bucketRequest(syncRules, 'global[]').bucket,
                     checksum: (c1 + c1 + c1 + c2) & 0xffffffff,
                     count: 4
                 }
             ]);
         }
-        catch (e_4) {
-            env_4.error = e_4;
-            env_4.hasError = true;
+        catch (e_8) {
+            env_8.error = e_8;
+            env_8.hasError = true;
         }
         finally {
-            const result_4 = __disposeResources(env_4);
-            if (result_4)
-                await result_4;
+            const result_8 = __disposeResources(env_8);
+            if (result_8)
+                await result_8;
         }
     });
     test('batch with overlapping replica ids', async () => {
-        const env_5 = { stack: [], error: void 0, hasError: false };
+        const env_9 = { stack: [], error: void 0, hasError: false };
         try {
             // This test checks that we get the correct output when processing rows with:
             // 1. changing replica ids
@@ -411,16 +699,17 @@ bucket_definitions:
             // It can break at two places:
             // 1. Not getting the correct "current_data" state for each operation.
             // 2. Output order not being correct.
-            const factory = __addDisposableResource(env_5, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
-`));
+`, { storageVersion }));
             const bucketStorage = factory.getInstance(syncRules);
             // Pre-setup
             const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 const sourceTable = TEST_TABLE;
                 await batch.save({
                     sourceTable,
@@ -540,18 +829,18 @@ bucket_definitions:
                 { op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
             ]);
         }
-        catch (e_5) {
-            env_5.error = e_5;
-            env_5.hasError = true;
+        catch (e_9) {
+            env_9.error = e_9;
+            env_9.hasError = true;
         }
         finally {
-            const result_5 = __disposeResources(env_5);
-            if (result_5)
-                await result_5;
+            const result_9 = __disposeResources(env_9);
+            if (result_9)
+                await result_9;
         }
     });
     test('changed data with replica identity full', async () => {
-        const env_6 = { stack: [], error: void 0, hasError: false };
+        const env_10 = { stack: [], error: void 0, hasError: false };
         try {
             function rid2(id, description) {
                 return getUuidReplicaIdentityBson({ id, description }, [
@@ -559,17 +848,20 @@ bucket_definitions:
                     { name: 'description', type: 'VARCHAR', typeId: 25 }
                 ]);
             }
-            const factory = __addDisposableResource(env_6, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
-            const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
+            const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
             // Pre-setup
             const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 await batch.save({
                     sourceTable,
                     tag: storage.SaveOperationTag.INSERT,
@@ -646,18 +938,18 @@ bucket_definitions:
                 }
             ]);
         }
-        catch (e_6) {
-            env_6.error = e_6;
-            env_6.hasError = true;
+        catch (e_10) {
+            env_10.error = e_10;
+            env_10.hasError = true;
         }
         finally {
-            const result_6 = __disposeResources(env_6);
-            if (result_6)
-                await result_6;
+            const result_10 = __disposeResources(env_10);
+            if (result_10)
+                await result_10;
         }
     });
     test('unchanged data with replica identity full', async () => {
-        const env_7 = { stack: [], error: void 0, hasError: false };
+        const env_11 = { stack: [], error: void 0, hasError: false };
         try {
             function rid2(id, description) {
                 return getUuidReplicaIdentityBson({ id, description }, [
@@ -665,17 +957,20 @@ bucket_definitions:
                     { name: 'description', type: 'VARCHAR', typeId: 25 }
                 ]);
             }
-            const factory = __addDisposableResource(env_7, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_11, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
-            const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
+            const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
             // Pre-setup
             const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 await batch.save({
                     sourceTable,
                     tag: storage.SaveOperationTag.INSERT,
@@ -745,32 +1040,35 @@ bucket_definitions:
                 }
             ]);
         }
-        catch (e_7) {
-            env_7.error = e_7;
-            env_7.hasError = true;
+        catch (e_11) {
+            env_11.error = e_11;
+            env_11.hasError = true;
         }
         finally {
-            const result_7 = __disposeResources(env_7);
-            if (result_7)
-                await result_7;
+            const result_11 = __disposeResources(env_11);
+            if (result_11)
+                await result_11;
         }
     });
     test('large batch', async () => {
-        const env_8 = { stack: [], error: void 0, hasError: false };
+        const env_12 = { stack: [], error: void 0, hasError: false };
         try {
             // Test syncing a batch of data that is small in count,
             // but large enough in size to be split over multiple returned batches.
             // The specific batch splits is an implementation detail of the storage driver,
             // and the test will have to updated when other implementations are added.
-            const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 const sourceTable = TEST_TABLE;
                 const largeDescription = '0123456789'.repeat(12_000_00);
                 await batch.save({
@@ -840,29 +1138,32 @@ bucket_definitions:
             expect(test_utils.getBatchData(batch3)).toEqual([]);
             expect(test_utils.getBatchMeta(batch3)).toEqual(null);
         }
-        catch (e_8) {
-            env_8.error = e_8;
-            env_8.hasError = true;
+        catch (e_12) {
+            env_12.error = e_12;
+            env_12.hasError = true;
         }
         finally {
-            const result_8 = __disposeResources(env_8);
-            if (result_8)
-                await result_8;
+            const result_12 = __disposeResources(env_12);
+            if (result_12)
+                await result_12;
         }
     });
     test('long batch', async () => {
-        const env_9 = { stack: [], error: void 0, hasError: false };
+        const env_13 = { stack: [], error: void 0, hasError: false };
         try {
             // Test syncing a batch of data that is limited by count.
-            const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 const sourceTable = TEST_TABLE;
                 for (let i = 1; i <= 6; i++) {
                     await batch.save({
@@ -908,21 +1209,21 @@ bucket_definitions:
             expect(test_utils.getBatchData(batch3)).toEqual([]);
             expect(test_utils.getBatchMeta(batch3)).toEqual(null);
         }
-        catch (e_9) {
-            env_9.error = e_9;
-            env_9.hasError = true;
+        catch (e_13) {
+            env_13.error = e_13;
+            env_13.hasError = true;
         }
         finally {
-            const result_9 = __disposeResources(env_9);
-            if (result_9)
-                await result_9;
+            const result_13 = __disposeResources(env_13);
+            if (result_13)
+                await result_13;
         }
     });
     describe('batch has_more', () => {
         const setup = async (options) => {
-            const env_10 = { stack: [], error: void 0, hasError: false };
+            const env_14 = { stack: [], error: void 0, hasError: false };
             try {
-                const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
+                const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
                 const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
   bucket_definitions:
     global1:
@@ -931,9 +1232,10 @@ bucket_definitions:
     global2:
       data:
         - SELECT id, description FROM test WHERE bucket = 'global2'
-  `));
+  `, { storageVersion }));
                 const bucketStorage = factory.getInstance(syncRules);
                 await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                    await batch.markAllSnapshotDone('1/1');
                     const sourceTable = TEST_TABLE;
                     for (let i = 1; i <= 10; i++) {
                         await batch.save({
@@ -956,21 +1258,21 @@ bucket_definitions:
                 ]), options));
                 return { syncRules, batch };
             }
-            catch (e_10) {
-                env_10.error = e_10;
-                env_10.hasError = true;
+            catch (e_14) {
+                env_14.error = e_14;
+                env_14.hasError = true;
             }
             finally {
-                const result_10 = __disposeResources(env_10);
-                if (result_10)
-                    await result_10;
+                const result_14 = __disposeResources(env_14);
+                if (result_14)
+                    await result_14;
             }
         };
         test('batch has_more (1)', async () => {
             const { batch, syncRules } = await setup({ limit: 5 });
             expect(batch.length).toEqual(2);
-            expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
-            expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
+            expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
+            expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
             expect(test_utils.getBatchData(batch[0])).toEqual([
                 { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
             ]);
@@ -994,8 +1296,8 @@ bucket_definitions:
         test('batch has_more (2)', async () => {
             const { batch, syncRules } = await setup({ limit: 11 });
             expect(batch.length).toEqual(2);
-            expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
-            expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
+            expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
+            expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
             expect(test_utils.getBatchData(batch[0])).toEqual([
                 { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
             ]);
@@ -1025,9 +1327,9 @@ bucket_definitions:
             // 50 bytes is more than 1 row, less than 2 rows
             const { batch, syncRules } = await setup({ limit: 3, chunkLimitBytes: 50 });
             expect(batch.length).toEqual(3);
-            expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
-            expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
-            expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
+            expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
+            expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
+            expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
             expect(test_utils.getBatchData(batch[0])).toEqual([
                 { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
             ]);
@@ -1055,9 +1357,9 @@ bucket_definitions:
         });
     });
     test('empty storage metrics', async () => {
-        const env_11 = { stack: [], error: void 0, hasError: false };
+        const env_15 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_11, await generateStorageFactory({ dropAll: true }), true);
+            const f = __addDisposableResource(env_15, await generateStorageFactory({ dropAll: true }), true);
             const metrics = await f.getStorageMetrics();
             expect(metrics).toEqual({
                 operations_size_bytes: 0,
@@ -1067,38 +1369,42 @@ bucket_definitions:
             const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
             const storage = f.getInstance(r.persisted_sync_rules);
             await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/0');
                 await batch.keepalive('1/0');
             });
             await f.getStorageMetrics();
         }
-        catch (e_11) {
-            env_11.error = e_11;
-            env_11.hasError = true;
+        catch (e_15) {
+            env_15.error = e_15;
+            env_15.hasError = true;
         }
         finally {
-            const result_11 = __disposeResources(env_11);
-            if (result_11)
-                await result_11;
+            const result_15 = __disposeResources(env_15);
+            if (result_15)
+                await result_15;
         }
     });
     test('op_id initialization edge case', async () => {
-        const env_12 = { stack: [], error: void 0, hasError: false };
+        const env_16 = { stack: [], error: void 0, hasError: false };
         try {
             // Test syncing a batch of data that is small in count,
             // but large enough in size to be split over multiple returned chunks.
             // Similar to the above test, but splits over 1MB chunks.
-            const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_16, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
   bucket_definitions:
     global:
       data:
         - SELECT id FROM test
         - SELECT id FROM test_ignore WHERE false
-  `));
+  `, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
-            const sourceTable = test_utils.makeTestTable('test', ['id']);
-            const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id']);
+            const sourceTable = test_utils.makeTestTable('test', ['id'], config);
+            const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id'], config);
             const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 // This saves a record to current_data, but not bucket_data.
                 // This causes a checkpoint to be created without increasing the op_id sequence.
                 await batch.save({
@@ -1125,29 +1431,32 @@ bucket_definitions:
             // we expect 0n and 1n, or 1n and 2n.
             expect(checkpoint2).toBeGreaterThan(checkpoint1);
         }
-        catch (e_12) {
-            env_12.error = e_12;
-            env_12.hasError = true;
+        catch (e_16) {
+            env_16.error = e_16;
+            env_16.hasError = true;
         }
         finally {
-            const result_12 = __disposeResources(env_12);
-            if (result_12)
-                await result_12;
+            const result_16 = __disposeResources(env_16);
+            if (result_16)
+                await result_16;
         }
     });
     test('unchanged checksums', async () => {
-        const env_13 = { stack: [], error: void 0, hasError: false };
+        const env_17 = { stack: [], error: void 0, hasError: false };
         try {
-            const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_17, await generateStorageFactory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT client_id as id, description FROM "%"
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
             const sourceTable = TEST_TABLE;
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 await batch.save({
                     sourceTable,
                     tag: storage.SaveOperationTag.INSERT,
@@ -1163,44 +1472,259 @@ bucket_definitions:
             const checksums = [
                 ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
             ];
-            expect(checksums).toEqual([{ bucket: bucketRequest(syncRules, 'global[]'), checksum: 1917136889, count: 1 }]);
+            expect(checksums).toEqual([
+                { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
+            ]);
             const checksums2 = [
                 ...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values()
             ];
-            expect(checksums2).toEqual([{ bucket: bucketRequest(syncRules, 'global[]'), checksum: 1917136889, count: 1 }]);
+            expect(checksums2).toEqual([
+                { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
+            ]);
         }
-        catch (e_13) {
-            env_13.error = e_13;
-            env_13.hasError = true;
+        catch (e_17) {
+            env_17.error = e_17;
+            env_17.hasError = true;
         }
         finally {
-            const result_13 = __disposeResources(env_13);
-            if (result_13)
-                await result_13;
+            const result_17 = __disposeResources(env_17);
+            if (result_17)
+                await result_17;
+        }
+    });
+    testChecksumBatching(config);
+    test('empty checkpoints (1)', async () => {
+        const env_18 = { stack: [], error: void 0, hasError: false };
+        try {
+            const factory = __addDisposableResource(env_18, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, { storageVersion }));
+            const bucketStorage = factory.getInstance(syncRules);
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
+                await batch.commit('1/1');
+                const cp1 = await bucketStorage.getCheckpoint();
+                expect(cp1.lsn).toEqual('1/1');
+                await batch.commit('2/1', { createEmptyCheckpoints: true });
+                const cp2 = await bucketStorage.getCheckpoint();
+                expect(cp2.lsn).toEqual('2/1');
+                await batch.keepalive('3/1');
+                const cp3 = await bucketStorage.getCheckpoint();
+                expect(cp3.lsn).toEqual('3/1');
+                // For the last one, we skip creating empty checkpoints
+                // This means the LSN stays at 3/1.
+                await batch.commit('4/1', { createEmptyCheckpoints: false });
+                const cp4 = await bucketStorage.getCheckpoint();
+                expect(cp4.lsn).toEqual('3/1');
+            });
+        }
+        catch (e_18) {
+            env_18.error = e_18;
+            env_18.hasError = true;
+        }
+        finally {
+            const result_18 = __disposeResources(env_18);
+            if (result_18)
+                await result_18;
+        }
+    });
+    test('empty checkpoints (2)', async () => {
+        const env_19 = { stack: [], error: void 0, hasError: false };
+        try {
+            const factory = __addDisposableResource(env_19, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, {
+                storageVersion
+            }));
+            const bucketStorage = factory.getInstance(syncRules);
+            const sourceTable = TEST_TABLE;
+            // We simulate two concurrent batches, but nesting is the easiest way to do this.
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch1) => {
+                await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch2) => {
+                    await batch1.markAllSnapshotDone('1/1');
+                    await batch1.commit('1/1');
+                    await batch1.commit('2/1', { createEmptyCheckpoints: false });
+                    const cp2 = await bucketStorage.getCheckpoint();
+                    expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
+                    await batch2.save({
+                        sourceTable,
+                        tag: storage.SaveOperationTag.INSERT,
+                        after: {
+                            id: 'test1',
+                            description: 'test1a'
+                        },
+                        afterReplicaId: test_utils.rid('test1')
+                    });
+                    // This simulates what happens on a snapshot processor.
+                    // This may later change to a flush() rather than commit().
+                    await batch2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
+                    const cp3 = await bucketStorage.getCheckpoint();
+                    expect(cp3.lsn).toEqual('1/1'); // Still unchanged
+                    // This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
+                    await batch1.commit('4/1', { createEmptyCheckpoints: false });
+                    const cp4 = await bucketStorage.getCheckpoint();
+                    expect(cp4.lsn).toEqual('4/1');
+                });
+            });
+        }
+        catch (e_19) {
+            env_19.error = e_19;
+            env_19.hasError = true;
+        }
+        finally {
+            const result_19 = __disposeResources(env_19);
+            if (result_19)
+                await result_19;
+        }
+    });
+    test('empty checkpoints (sync rule activation)', async () => {
+        const env_20 = { stack: [], error: void 0, hasError: false };
+        try {
+            const factory = __addDisposableResource(env_20, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, {
+                storageVersion
+            }));
+            const bucketStorage = factory.getInstance(syncRules);
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                const result = await batch.commit('1/1', { createEmptyCheckpoints: false });
+                expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
+                // Snapshot is only valid once we reach 3/1
+                await batch.markAllSnapshotDone('3/1');
+            });
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                // 2/1 < 3/1 - snapshot not valid yet, block checkpoint
+                const result = await batch.commit('2/1', { createEmptyCheckpoints: false });
+                expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
+            });
+            // No empty checkpoint should be created by the commit above.
+            const cp1 = await bucketStorage.getCheckpoint();
+            expect(cp1.lsn).toEqual(null);
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                // After this commit, the snapshot should be valid.
+                // We specifically check that this is done even if createEmptyCheckpoints: false.
+                const result = await batch.commit('3/1', { createEmptyCheckpoints: false });
+                expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: true });
+            });
+            // Now, the checkpoint should advance the sync rules active.
+            const cp2 = await bucketStorage.getCheckpoint();
+            expect(cp2.lsn).toEqual('3/1');
+            const activeSyncRules = await factory.getActiveSyncRulesContent();
+            expect(activeSyncRules?.id).toEqual(syncRules.id);
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                // At this point, it should be a truely empty checkpoint
+                const result = await batch.commit('4/1', { createEmptyCheckpoints: false });
+                expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: false });
+            });
+            // Unchanged
+            const cp3 = await bucketStorage.getCheckpoint();
+            expect(cp3.lsn).toEqual('3/1');
+        }
+        catch (e_20) {
+            env_20.error = e_20;
+            env_20.hasError = true;
+        }
+        finally {
+            const result_20 = __disposeResources(env_20);
+            if (result_20)
+                await result_20;
+        }
+    });
+    test.runIf(storageVersion >= 3)('deleting while streaming', async () => {
+        const env_21 = { stack: [], error: void 0, hasError: false };
+        try {
+            const factory = __addDisposableResource(env_21, await generateStorageFactory(), true);
+            const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`, {
+                storageVersion
+            }));
+            const bucketStorage = factory.getInstance(syncRules);
+            const sourceTable = TEST_TABLE;
+            // We simulate two concurrent batches, and nesting is the easiest way to do this.
+            // For this test, we assume that we start with a row "test1", which is picked up by a snapshot
+            // query, right before the delete is streamed. But the snapshot query is only persisted _after_
+            // the delete is streamed, and we need to ensure that the streamed delete takes precedence.
+            await bucketStorage.startBatch({ ...test_utils.BATCH_OPTIONS, skipExistingRows: true }, async (snapshotBatch) => {
+                await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (streamingBatch) => {
+                    streamingBatch.save({
+                        sourceTable,
+                        tag: storage.SaveOperationTag.DELETE,
+                        before: {
+                            id: 'test1'
+                        },
+                        beforeReplicaId: test_utils.rid('test1')
+                    });
+                    await streamingBatch.commit('2/1');
+                    await snapshotBatch.save({
+                        sourceTable,
+                        tag: storage.SaveOperationTag.INSERT,
+                        after: {
+                            id: 'test1',
+                            description: 'test1a'
+                        },
+                        afterReplicaId: test_utils.rid('test1')
+                    });
+                    await snapshotBatch.markAllSnapshotDone('3/1');
+                    await snapshotBatch.commit('1/1');
+                    await streamingBatch.keepalive('3/1');
+                });
+            });
+            const cp = await bucketStorage.getCheckpoint();
+            expect(cp.lsn).toEqual('3/1');
+            const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
+            expect(data).toEqual([]);
+        }
+        catch (e_21) {
+            env_21.error = e_21;
+            env_21.hasError = true;
+        }
+        finally {
+            const result_21 = __disposeResources(env_21);
+            if (result_21)
+                await result_21;
         }
     });
-    testChecksumBatching(generateStorageFactory);
 }
 /**
  * This specifically tests an issue we ran into with MongoDB storage.
  *
  * Exposed as a separate test so we can test with more storage parameters.
  */
-export function testChecksumBatching(generateStorageFactory) {
+export function testChecksumBatching(config) {
+    const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
     test('checksums for multiple buckets', async () => {
-        const env_14 = { stack: [], error: void 0, hasError: false };
+        const env_22 = { stack: [], error: void 0, hasError: false };
         try {
-            const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
+            const factory = __addDisposableResource(env_22, await config.factory(), true);
             const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
 bucket_definitions:
   user:
     parameters: select request.user_id() as user_id
     data:
       - select id, description from test where user_id = bucket.user_id
-`));
+`, {
+                storageVersion
+            }));
             const bucketStorage = factory.getInstance(syncRules);
-            const sourceTable = TEST_TABLE;
+            const sourceTable = test_utils.makeTestTable('test', ['id'], config);
             await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                await batch.markAllSnapshotDone('1/1');
                 for (let u of ['u1', 'u2', 'u3', 'u4']) {
                     for (let t of ['t1', 't2', 't3', 't4']) {
                         const id = `${t}_${u}`;
@@ -1224,20 +1748,20 @@ bucket_definitions:
             const checksums = [...(await bucketStorage.getChecksums(checkpoint, buckets)).values()];
             checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
             expect(checksums).toEqual([
-                { bucket: bucketRequest(syncRules, 'user["u1"]'), count: 4, checksum: 346204588 },
-                { bucket: bucketRequest(syncRules, 'user["u2"]'), count: 4, checksum: 5261081 },
-                { bucket: bucketRequest(syncRules, 'user["u3"]'), count: 4, checksum: 134760718 },
-                { bucket: bucketRequest(syncRules, 'user["u4"]'), count: 4, checksum: -302639724 }
+                { bucket: bucketRequest(syncRules, 'user["u1"]').bucket, count: 4, checksum: 346204588 },
+                { bucket: bucketRequest(syncRules, 'user["u2"]').bucket, count: 4, checksum: 5261081 },
+                { bucket: bucketRequest(syncRules, 'user["u3"]').bucket, count: 4, checksum: 134760718 },
+                { bucket: bucketRequest(syncRules, 'user["u4"]').bucket, count: 4, checksum: -302639724 }
             ]);
         }
-        catch (e_14) {
-            env_14.error = e_14;
-            env_14.hasError = true;
+        catch (e_22) {
+            env_22.error = e_22;
+            env_22.hasError = true;
         }
         finally {
-            const result_14 = __disposeResources(env_14);
-            if (result_14)
-                await result_14;
+            const result_22 = __disposeResources(env_22);
+            if (result_22)
+                await result_22;
         }
     });
 }