@powersync/service-core 1.10.1 → 1.10.3

package/test/dist/data_storage.test.js

@@ -0,0 +1,1306 @@
+import { SaveOperationTag } from '@/storage/BucketStorage.js';
+import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
+import { RequestParameters } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+import { fromAsync, oneFromAsync } from './stream_utils.js';
+import { BATCH_OPTIONS, getBatchData, getBatchMeta, makeTestTable, MONGO_STORAGE_FACTORY, PARSE_OPTIONS, rid, testRules } from './util.js';
+const TEST_TABLE = makeTestTable('test', ['id']);
+describe('store - mongodb', function () {
+    defineDataStorageTests(MONGO_STORAGE_FACTORY);
+});
+function defineDataStorageTests(factory) {
+    test('save and load parameters', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
+    data: [] 
+    `);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 't2',
+                    id1: 'user3',
+                    id2: 'user4',
+                    group_id: 'group2a'
+                },
+                afterReplicaId: rid('t2')
+            });
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 't1',
+                    id1: 'user1',
+                    id2: 'user2',
+                    group_id: 'group1a'
+                },
+                afterReplicaId: rid('t1')
+            });
+        });
+        const parameters = await storage.getParameterSets(result.flushed_op, [['mybucket', '1', 'user1']]);
+        expect(parameters).toEqual([
+            {
+                group_id: 'group1a'
+            }
+        ]);
+    });
+    test('it should use the latest version', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT group_id FROM test WHERE id = token_parameters.user_id
+    data: [] 
+    `);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result1 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'user1',
+                    group_id: 'group1'
+                },
+                afterReplicaId: rid('user1')
+            });
+        });
+        const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'user1',
+                    group_id: 'group2'
+                },
+                afterReplicaId: rid('user1')
+            });
+        });
+        const parameters = await storage.getParameterSets(result2.flushed_op, [['mybucket', '1', 'user1']]);
+        expect(parameters).toEqual([
+            {
+                group_id: 'group2'
+            }
+        ]);
+        // Use the checkpoint to get older data if relevant
+        const parameters2 = await storage.getParameterSets(result1.flushed_op, [['mybucket', '1', 'user1']]);
+        expect(parameters2).toEqual([
+            {
+                group_id: 'group1'
+            }
+        ]);
+    });
+    test('it should use the latest version after updates', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT id AS todo_id
+        FROM todos
+        WHERE list_id IN token_parameters.list_id
+    data: [] 
+    `);
+        const storage = (await factory()).getInstance(sync_rules);
+        const table = makeTestTable('todos', ['id', 'list_id']);
+        await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            // Create two todos which initially belong to different lists
+            await batch.save({
+                sourceTable: table,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'todo1',
+                    list_id: 'list1'
+                },
+                afterReplicaId: rid('todo1')
+            });
+            await batch.save({
+                sourceTable: table,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'todo2',
+                    list_id: 'list2'
+                },
+                afterReplicaId: rid('todo2')
+            });
+        });
+        const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            // Update the second todo item to now belong to list 1
+            await batch.save({
+                sourceTable: table,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 'todo2',
+                    list_id: 'list1'
+                },
+                afterReplicaId: rid('todo2')
+            });
+        });
+        // We specifically request the todo_ids for both lists.
+        // There removal operation for the association of `list2`::`todo2` should not interfere with the new
+        // association of `list1`::`todo2`
+        const parameters = await storage.getParameterSets(BigInt(result2.flushed_op).toString(), [
+            ['mybucket', '1', 'list1'],
+            ['mybucket', '1', 'list2']
+        ]);
+        expect(parameters.sort((a, b) => a.todo_id.localeCompare(b.todo_id))).toEqual([
+            {
+                todo_id: 'todo1'
+            },
+            {
+                todo_id: 'todo2'
+            }
+        ]);
+    });
+    test('save and load parameters with different number types', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT group_id FROM test WHERE n1 = token_parameters.n1 and f2 = token_parameters.f2 and f3 = token_parameters.f3
+    data: []
+    `);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 't1',
+                    group_id: 'group1',
+                    n1: 314n,
+                    f2: 314,
+                    f3: 3.14
+                },
+                afterReplicaId: rid('t1')
+            });
+        });
+        const TEST_PARAMS = { group_id: 'group1' };
+        const checkpoint = result.flushed_op;
+        const parameters1 = await storage.getParameterSets(checkpoint, [['mybucket', '1', 314n, 314, 3.14]]);
+        expect(parameters1).toEqual([TEST_PARAMS]);
+        const parameters2 = await storage.getParameterSets(checkpoint, [['mybucket', '1', 314, 314n, 3.14]]);
+        expect(parameters2).toEqual([TEST_PARAMS]);
+        const parameters3 = await storage.getParameterSets(checkpoint, [['mybucket', '1', 314n, 314, 3]]);
+        expect(parameters3).toEqual([]);
+    });
+    test('save and load parameters with large numbers', async () => {
+        // This ensures serialization / deserialization of "current_data" is done correctly.
+        // This specific case tested here cannot happen with postgres in practice, but we still
+        // test this to ensure correct deserialization.
+        const sync_rules = testRules(`
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT group_id FROM test WHERE n1 = token_parameters.n1
+    data: []
+    `);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 't1',
+                    group_id: 'group1',
+                    n1: 1152921504606846976n // 2^60
+                },
+                afterReplicaId: rid('t1')
+            });
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 't1',
+                    group_id: 'group1',
+                    // Simulate a TOAST value, even though it can't happen for values like this
+                    // in practice.
+                    n1: undefined
+                },
+                afterReplicaId: rid('t1')
+            });
+        });
+        const TEST_PARAMS = { group_id: 'group1' };
+        const checkpoint = result.flushed_op;
+        const parameters1 = await storage.getParameterSets(checkpoint, [['mybucket', '1', 1152921504606846976n]]);
+        expect(parameters1).toEqual([TEST_PARAMS]);
+    });
+    test('removing row', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1'
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.DELETE,
+                beforeReplicaId: rid('test1')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+        const data = batch[0].batch.data.map((d) => {
+            return {
+                op: d.op,
+                object_id: d.object_id,
+                checksum: d.checksum
+            };
+        });
+        const c1 = 2871785649;
+        const c2 = 2872534815;
+        expect(data).toEqual([
+            { op: 'PUT', object_id: 'test1', checksum: c1 },
+            { op: 'REMOVE', object_id: 'test1', checksum: c2 }
+        ]);
+        const checksums = [...(await storage.getChecksums(checkpoint, ['global[]'])).values()];
+        expect(checksums).toEqual([
+            {
+                bucket: 'global[]',
+                checksum: (c1 + c2) & 0xffffffff,
+                count: 2
+            }
+        ]);
+    });
+    test('save and load parameters with workspaceId', async () => {
+        const WORKSPACE_TABLE = makeTestTable('workspace', ['id']);
+        const sync_rules_content = testRules(`
+bucket_definitions:
+    by_workspace:
+      parameters:
+        - SELECT id as workspace_id FROM workspace WHERE
+          workspace."userId" = token_parameters.user_id
+      data: []
+    `);
+        const sync_rules = sync_rules_content.parsed(PARSE_OPTIONS).sync_rules;
+        const storage = (await factory()).getInstance(sync_rules_content);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace1',
+                    userId: 'u1'
+                },
+                afterReplicaId: rid('workspace1')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const parameters = new RequestParameters({ sub: 'u1' }, {});
+        const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
+        const lookups = q1.getLookups(parameters);
+        expect(lookups).toEqual([['by_workspace', '1', 'u1']]);
+        const parameter_sets = await storage.getParameterSets(checkpoint, lookups);
+        expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
+        const buckets = await sync_rules.queryBucketIds({
+            getParameterSets(lookups) {
+                return storage.getParameterSets(checkpoint, lookups);
+            },
+            parameters
+        });
+        expect(buckets).toEqual(['by_workspace["workspace1"]']);
+    });
+    test('save and load parameters with dynamic global buckets', async () => {
+        const WORKSPACE_TABLE = makeTestTable('workspace');
+        const sync_rules_content = testRules(`
+bucket_definitions:
+    by_public_workspace:
+      parameters:
+        - SELECT id as workspace_id FROM workspace WHERE
+          workspace.visibility = 'public'
+      data: []
+    `);
+        const sync_rules = sync_rules_content.parsed(PARSE_OPTIONS).sync_rules;
+        const storage = (await factory()).getInstance(sync_rules_content);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace1',
+                    visibility: 'public'
+                },
+                afterReplicaId: rid('workspace1')
+            });
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace2',
+                    visibility: 'private'
+                },
+                afterReplicaId: rid('workspace2')
+            });
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace3',
+                    visibility: 'public'
+                },
+                afterReplicaId: rid('workspace3')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const parameters = new RequestParameters({ sub: 'unknown' }, {});
+        const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
+        const lookups = q1.getLookups(parameters);
+        expect(lookups).toEqual([['by_public_workspace', '1']]);
+        const parameter_sets = await storage.getParameterSets(checkpoint, lookups);
+        parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
+        expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
+        const buckets = await sync_rules.queryBucketIds({
+            getParameterSets(lookups) {
+                return storage.getParameterSets(checkpoint, lookups);
+            },
+            parameters
+        });
+        buckets.sort();
+        expect(buckets).toEqual(['by_public_workspace["workspace1"]', 'by_public_workspace["workspace3"]']);
+    });
+    test('multiple parameter queries', async () => {
+        const WORKSPACE_TABLE = makeTestTable('workspace');
+        const sync_rules_content = testRules(`
+bucket_definitions:
+    by_workspace:
+      parameters:
+        - SELECT id as workspace_id FROM workspace WHERE
+          workspace.visibility = 'public'
+        - SELECT id as workspace_id FROM workspace WHERE
+            workspace.user_id = token_parameters.user_id
+      data: []
+    `);
+        const sync_rules = sync_rules_content.parsed(PARSE_OPTIONS).sync_rules;
+        const storage = (await factory()).getInstance(sync_rules_content);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace1',
+                    visibility: 'public'
+                },
+                afterReplicaId: rid('workspace1')
+            });
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace2',
+                    visibility: 'private'
+                },
+                afterReplicaId: rid('workspace2')
+            });
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace3',
+                    user_id: 'u1',
+                    visibility: 'private'
+                },
+                afterReplicaId: rid('workspace3')
+            });
+            await batch.save({
+                sourceTable: WORKSPACE_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'workspace4',
+                    user_id: 'u2',
+                    visibility: 'private'
+                },
+                afterReplicaId: rid('workspace4')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const parameters = new RequestParameters({ sub: 'u1' }, {});
+        // Test intermediate values - could be moved to sync_rules.test.ts
+        const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
+        const lookups1 = q1.getLookups(parameters);
+        expect(lookups1).toEqual([['by_workspace', '1']]);
+        const parameter_sets1 = await storage.getParameterSets(checkpoint, lookups1);
+        parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
+        expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]);
+        const q2 = sync_rules.bucket_descriptors[0].parameter_queries[1];
+        const lookups2 = q2.getLookups(parameters);
+        expect(lookups2).toEqual([['by_workspace', '2', 'u1']]);
+        const parameter_sets2 = await storage.getParameterSets(checkpoint, lookups2);
+        parameter_sets2.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
+        expect(parameter_sets2).toEqual([{ workspace_id: 'workspace3' }]);
+        // Test final values - the important part
+        const buckets = await sync_rules.queryBucketIds({
+            getParameterSets(lookups) {
+                return storage.getParameterSets(checkpoint, lookups);
+            },
+            parameters
+        });
+        buckets.sort();
+        expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']);
+    });
+    test('changing client ids', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT client_id as id, description FROM "%"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        const sourceTable = TEST_TABLE;
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    client_id: 'client1a',
+                    description: 'test1a'
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 'test1',
+                    client_id: 'client1b',
+                    description: 'test1b'
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test2',
+                    client_id: 'client2',
+                    description: 'test2'
+                },
+                afterReplicaId: rid('test2')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+        const data = batch[0].batch.data.map((d) => {
+            return {
+                op: d.op,
+                object_id: d.object_id
+            };
+        });
+        expect(data).toEqual([
+            { op: 'PUT', object_id: 'client1a' },
+            { op: 'PUT', object_id: 'client1b' },
+            { op: 'REMOVE', object_id: 'client1a' },
+            { op: 'PUT', object_id: 'client2' }
+        ]);
+    });
+    test('re-apply delete', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1'
+                },
+                afterReplicaId: rid('test1')
+            });
+        });
+        await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.DELETE,
+                beforeReplicaId: rid('test1')
+            });
+        });
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.DELETE,
+                beforeReplicaId: rid('test1')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+        const data = batch[0].batch.data.map((d) => {
+            return {
+                op: d.op,
+                object_id: d.object_id,
+                checksum: d.checksum
+            };
+        });
+        const c1 = 2871785649;
+        const c2 = 2872534815;
+        expect(data).toEqual([
+            { op: 'PUT', object_id: 'test1', checksum: c1 },
+            { op: 'REMOVE', object_id: 'test1', checksum: c2 }
+        ]);
+        const checksums = [...(await storage.getChecksums(checkpoint, ['global[]'])).values()];
+        expect(checksums).toEqual([
+            {
+                bucket: 'global[]',
+                checksum: (c1 + c2) & 0xffffffff,
+                count: 2
+            }
+        ]);
+    });
+    test('re-apply update + delete', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1'
+                },
+                afterReplicaId: rid('test1')
+            });
+        });
+        await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 'test1',
+                    description: undefined
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 'test1',
+                    description: undefined
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.DELETE,
+                beforeReplicaId: rid('test1')
+            });
+        });
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 'test1',
+                    description: undefined
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 'test1',
+                    description: undefined
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.DELETE,
+                beforeReplicaId: rid('test1')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+        const data = batch[0].batch.data.map((d) => {
+            return {
+                op: d.op,
+                object_id: d.object_id,
+                checksum: d.checksum
+            };
+        });
+        const c1 = 2871785649;
+        const c2 = 2872534815;
+        expect(data).toEqual([
+            { op: 'PUT', object_id: 'test1', checksum: c1 },
+            { op: 'PUT', object_id: 'test1', checksum: c1 },
+            { op: 'PUT', object_id: 'test1', checksum: c1 },
+            { op: 'REMOVE', object_id: 'test1', checksum: c2 }
+        ]);
+        const checksums = [...(await storage.getChecksums(checkpoint, ['global[]'])).values()];
+        expect(checksums).toEqual([
+            {
+                bucket: 'global[]',
+                checksum: (c1 + c1 + c1 + c2) & 0xffffffff,
+                count: 4
+            }
+        ]);
+    });
+    test('truncate parameters', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
+    data: []
+    `);
+        const storage = (await factory()).getInstance(sync_rules);
+        await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 't2',
+                    id1: 'user3',
+                    id2: 'user4',
+                    group_id: 'group2a'
+                },
+                afterReplicaId: rid('t2')
+            });
+            await batch.truncate([TEST_TABLE]);
+        });
+        const { checkpoint } = await storage.getCheckpoint();
+        const parameters = await storage.getParameterSets(checkpoint, [['mybucket', '1', 'user1']]);
+        expect(parameters).toEqual([]);
+    });
+    test('batch with overlapping replica ids', async () => {
+        // This test checks that we get the correct output when processing rows with:
+        // 1. changing replica ids
+        // 2. overlapping with replica ids of other rows in the same transaction (at different times)
+        // If operations are not processing in input order, this breaks easily.
+        // It can break at two places:
+        // 1. Not getting the correct "current_data" state for each operation.
+        // 2. Output order not being correct.
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        // Pre-setup
+        const result1 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1a'
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test2',
+                    description: 'test2a'
+                },
+                afterReplicaId: rid('test2')
+            });
+        });
+        const checkpoint1 = result1?.flushed_op ?? '0';
+        // Test batch
+        const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            // b
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1b'
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                before: {
+                    id: 'test1'
+                },
+                beforeReplicaId: rid('test1'),
+                after: {
+                    id: 'test2',
+                    description: 'test2b'
+                },
+                afterReplicaId: rid('test2')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                before: {
+                    id: 'test2'
+                },
+                beforeReplicaId: rid('test2'),
+                after: {
+                    id: 'test3',
+                    description: 'test3b'
+                },
+                afterReplicaId: rid('test3')
+            });
+            // c
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                after: {
+                    id: 'test2',
+                    description: 'test2c'
+                },
+                afterReplicaId: rid('test2')
+            });
+            // d
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test4',
+                    description: 'test4d'
+                },
+                afterReplicaId: rid('test4')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                before: {
+                    id: 'test4'
+                },
+                beforeReplicaId: rid('test4'),
+                after: {
+                    id: 'test5',
+                    description: 'test5d'
+                },
+                afterReplicaId: rid('test5')
+            });
+        });
+        const checkpoint2 = result2.flushed_op;
+        const batch = await fromAsync(storage.getBucketDataBatch(checkpoint2, new Map([['global[]', checkpoint1]])));
+        const data = batch[0].batch.data.map((d) => {
+            return {
+                op: d.op,
+                object_id: d.object_id,
+                data: d.data
+            };
+        });
+        // Operations must be in this order
+        expect(data).toEqual([
+            // b
+            { op: 'PUT', object_id: 'test1', data: JSON.stringify({ id: 'test1', description: 'test1b' }) },
+            { op: 'REMOVE', object_id: 'test1', data: null },
+            { op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2b' }) },
+            { op: 'REMOVE', object_id: 'test2', data: null },
+            { op: 'PUT', object_id: 'test3', data: JSON.stringify({ id: 'test3', description: 'test3b' }) },
+            // c
+            { op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2c' }) },
+            // d
+            { op: 'PUT', object_id: 'test4', data: JSON.stringify({ id: 'test4', description: 'test4d' }) },
+            { op: 'REMOVE', object_id: 'test4', data: null },
+            { op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
+        ]);
+    });
+    test('changed data with replica identity full', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`);
+        function rid2(id, description) {
+            return getUuidReplicaIdentityBson({ id, description }, [
+                { name: 'id', type: 'VARCHAR', typeId: 25 },
+                { name: 'description', type: 'VARCHAR', typeId: 25 }
+            ]);
+        }
+        const storage = (await factory()).getInstance(sync_rules);
+        const sourceTable = makeTestTable('test', ['id', 'description']);
+        // Pre-setup
+        const result1 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1a'
+                },
+                afterReplicaId: rid2('test1', 'test1a')
+            });
+        });
+        const checkpoint1 = result1?.flushed_op ?? '0';
+        const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            // Unchanged, but has a before id
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                before: {
+                    id: 'test1',
+                    description: 'test1a'
+                },
+                beforeReplicaId: rid2('test1', 'test1a'),
+                after: {
+                    id: 'test1',
+                    description: 'test1b'
+                },
+                afterReplicaId: rid2('test1', 'test1b')
+            });
+        });
+        const result3 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            // Delete
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.DELETE,
+                before: {
+                    id: 'test1',
+                    description: 'test1b'
+                },
+                beforeReplicaId: rid2('test1', 'test1b'),
+                after: undefined
+            });
+        });
+        const checkpoint3 = result3.flushed_op;
+        const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
+        const data = batch[0].batch.data.map((d) => {
+            return {
+                op: d.op,
+                object_id: d.object_id,
+                data: d.data,
+                subkey: d.subkey
+            };
+        });
+        // Operations must be in this order
+        expect(data).toEqual([
+            // 2
+            // The REMOVE is expected because the subkey changes
+            {
+                op: 'REMOVE',
+                object_id: 'test1',
+                data: null,
+                subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+            },
+            {
+                op: 'PUT',
+                object_id: 'test1',
+                data: JSON.stringify({ id: 'test1', description: 'test1b' }),
+                subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
+            },
+            // 3
+            {
+                op: 'REMOVE',
+                object_id: 'test1',
+                data: null,
+                subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
+            }
+        ]);
+    });
+    test('unchanged data with replica identity full', async () => {
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`);
+        function rid2(id, description) {
+            return getUuidReplicaIdentityBson({ id, description }, [
+                { name: 'id', type: 'VARCHAR', typeId: 25 },
+                { name: 'description', type: 'VARCHAR', typeId: 25 }
+            ]);
+        }
+        const storage = (await factory()).getInstance(sync_rules);
+        const sourceTable = makeTestTable('test', ['id', 'description']);
+        // Pre-setup
+        const result1 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1a'
+                },
+                afterReplicaId: rid2('test1', 'test1a')
+            });
+        });
+        const checkpoint1 = result1?.flushed_op ?? '0';
+        const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            // Unchanged, but has a before id
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.UPDATE,
+                before: {
+                    id: 'test1',
+                    description: 'test1a'
+                },
+                beforeReplicaId: rid2('test1', 'test1a'),
+                after: {
+                    id: 'test1',
+                    description: 'test1a'
+                },
+                afterReplicaId: rid2('test1', 'test1a')
+            });
+        });
+        const result3 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            // Delete
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.DELETE,
+                before: {
+                    id: 'test1',
+                    description: 'test1a'
+                },
+                beforeReplicaId: rid2('test1', 'test1a'),
+                after: undefined
+            });
+        });
+        const checkpoint3 = result3.flushed_op;
+        const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
+        const data = batch[0].batch.data.map((d) => {
+            return {
+                op: d.op,
+                object_id: d.object_id,
+                data: d.data,
+                subkey: d.subkey
+            };
+        });
+        // Operations must be in this order
+        expect(data).toEqual([
+            // 2
+            {
+                op: 'PUT',
+                object_id: 'test1',
+                data: JSON.stringify({ id: 'test1', description: 'test1a' }),
+                subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+            },
+            // 3
+            {
+                op: 'REMOVE',
+                object_id: 'test1',
+                data: null,
+                subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+            }
+        ]);
+    });
+    test('large batch', async () => {
+        // Test syncing a batch of data that is small in count,
+        // but large enough in size to be split over multiple returned batches.
+        // The specific batch splits is an implementation detail of the storage driver,
+        // and the test will have to updated when other implementations are added.
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            const largeDescription = '0123456789'.repeat(1200000);
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1'
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'large1',
+                    description: largeDescription
+                },
+                afterReplicaId: rid('large1')
+            });
+            // Large enough to split the returned batch
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'large2',
+                    description: largeDescription
+                },
+                afterReplicaId: rid('large2')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test3',
+                    description: 'test3'
+                },
+                afterReplicaId: rid('test3')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const options = {
+            chunkLimitBytes: 16 * 1024 * 1024
+        };
+        const batch1 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options));
+        expect(getBatchData(batch1)).toEqual([
+            { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
+            { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
+        ]);
+        expect(getBatchMeta(batch1)).toEqual({
+            after: '0',
+            has_more: true,
+            next_after: '2'
+        });
+        const batch2 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options));
+        expect(getBatchData(batch2)).toEqual([
+            { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
+            { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+        ]);
+        expect(getBatchMeta(batch2)).toEqual({
+            after: '2',
+            has_more: false,
+            next_after: '4'
+        });
+        const batch3 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options));
+        expect(getBatchData(batch3)).toEqual([]);
+        expect(getBatchMeta(batch3)).toEqual(null);
+    });
+    test('large batch (2)', async () => {
+        // Test syncing a batch of data that is small in count,
+        // but large enough in size to be split over multiple returned chunks.
+        // Similar to the above test, but splits over 1MB chunks.
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            const largeDescription = '0123456789'.repeat(200000);
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test1',
+                    description: 'test1'
+                },
+                afterReplicaId: rid('test1')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'large1',
+                    description: largeDescription
+                },
+                afterReplicaId: rid('large1')
+            });
+            // Large enough to split the returned batch
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'large2',
+                    description: largeDescription
+                },
+                afterReplicaId: rid('large2')
+            });
+            await batch.save({
+                sourceTable,
+                tag: SaveOperationTag.INSERT,
+                after: {
+                    id: 'test3',
+                    description: 'test3'
+                },
+                afterReplicaId: rid('test3')
+            });
+        });
+        const checkpoint = result.flushed_op;
+        const options = {};
+        const batch1 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options));
+        expect(getBatchData(batch1)).toEqual([
+            { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
+            { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+        ]);
+        expect(getBatchMeta(batch1)).toEqual({
+            after: '0',
+            has_more: true,
+            next_after: '2'
+        });
+        const batch2 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options));
+        expect(getBatchData(batch2)).toEqual([{ op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }]);
+        expect(getBatchMeta(batch2)).toEqual({
+            after: '2',
+            has_more: true,
+            next_after: '3'
+        });
+        const batch3 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options));
+        expect(getBatchData(batch3)).toEqual([{ op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }]);
+        expect(getBatchMeta(batch3)).toEqual({
+            after: '3',
+            has_more: false,
+            next_after: '4'
+        });
+    });
+    test('long batch', async () => {
+        // Test syncing a batch of data that is limited by count.
+        const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "%"
+`);
+        const storage = (await factory()).getInstance(sync_rules);
+        const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            const sourceTable = TEST_TABLE;
+            for (let i = 1; i <= 6; i++) {
+                await batch.save({
+                    sourceTable,
+                    tag: SaveOperationTag.INSERT,
+                    after: {
+                        id: `test${i}`,
+                        description: `test${i}`
+                    },
+                    afterReplicaId: `test${i}`
+                });
+            }
+        });
+        const checkpoint = result.flushed_op;
+        const batch1 = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), { limit: 4 }));
+        expect(getBatchData(batch1)).toEqual([
+            { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
+            { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
+            { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
+            { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
+        ]);
+        expect(getBatchMeta(batch1)).toEqual({
+            after: '0',
+            has_more: true,
+            next_after: '4'
+        });
+        const batch2 = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1.batch.next_after]]), {
+            limit: 4
+        }));
+        expect(getBatchData(batch2)).toEqual([
+            { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
+            { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
+        ]);
+        expect(getBatchMeta(batch2)).toEqual({
+            after: '4',
+            has_more: false,
+            next_after: '6'
+        });
+        const batch3 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2.batch.next_after]]), {
+            limit: 4
+        }));
+        expect(getBatchData(batch3)).toEqual([]);
+        expect(getBatchMeta(batch3)).toEqual(null);
+    });
+    test('batch should be disposed automatically', async () => {
+        const sync_rules = testRules(`
+      bucket_definitions:
+        global:
+          data: [] 
+          `);
+        const storage = (await factory()).getInstance(sync_rules);
+        let isDisposed = false;
+        await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+            batch.registerListener({
+                disposed: () => {
+                    isDisposed = true;
+                }
+            });
+        });
+        expect(isDisposed).true;
+        isDisposed = false;
+        let errorCaught = false;
+        try {
+            await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+                batch.registerListener({
+                    disposed: () => {
+                        isDisposed = true;
+                    }
+                });
+                throw new Error(`Testing exceptions`);
+            });
+        }
+        catch (ex) {
+            errorCaught = true;
+            expect(ex.message.includes('Testing')).true;
+        }
+        expect(errorCaught).true;
+        expect(isDisposed).true;
+    });
+    test('empty storage metrics', async () => {
+        const f = await factory({ dropAll: true });
+        const metrics = await f.getStorageMetrics();
+        expect(metrics).toEqual({
+            operations_size_bytes: 0,
+            parameters_size_bytes: 0,
+            replication_size_bytes: 0
+        });
+        const r = await f.configureSyncRules('bucket_definitions: {}');
+        const storage = f.getInstance(r.persisted_sync_rules);
+        await storage.autoActivate();
+        const metrics2 = await f.getStorageMetrics();
+        expect(metrics2).toEqual({
+            operations_size_bytes: 0,
+            parameters_size_bytes: 0,
+            replication_size_bytes: 0
+        });
+    });
+    test('invalidate cached parsed sync rules', async () => {
+        const sync_rules_content = testRules(`
+bucket_definitions:
+    by_workspace:
+      parameters:
+        - SELECT id as workspace_id FROM workspace WHERE
+          workspace."userId" = token_parameters.user_id
+      data: []
+    `);
+        const bucketStorageFactory = await factory();
+        const syncBucketStorage = bucketStorageFactory.getInstance(sync_rules_content);
+        const parsedSchema1 = syncBucketStorage.getParsedSyncRules({
+            defaultSchema: 'public'
+        });
+        const parsedSchema2 = syncBucketStorage.getParsedSyncRules({
+            defaultSchema: 'public'
+        });
+        // These should be cached, this will be the same instance
+        expect(parsedSchema2).equals(parsedSchema1);
+        expect(parsedSchema1.getSourceTables()[0].schema).equals('public');
+        const parsedSchema3 = syncBucketStorage.getParsedSyncRules({
+            defaultSchema: 'databasename'
+        });
+        // The cache should not be used
+        expect(parsedSchema3).not.equals(parsedSchema2);
+        expect(parsedSchema3.getSourceTables()[0].schema).equals('databasename');
+    });
+}
+//# sourceMappingURL=data_storage.test.js.map