@powersync/service-core-tests 0.13.1 → 0.14.0

package/src/tests/register-data-storage-data-tests.ts

@@ -1,7 +1,13 @@
-import { BucketDataBatchOptions, getUuidReplicaIdentityBson, OplogEntry, storage } from '@powersync/service-core';
+import {
+  BucketDataBatchOptions,
+  getUuidReplicaIdentityBson,
+  OplogEntry,
+  storage,
+  updateSyncRulesFromYaml
+} from '@powersync/service-core';
 import { describe, expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
-import { TEST_TABLE } from './util.js';
+import { bucketRequest, bucketRequestMap, bucketRequests, TEST_TABLE } from './util.js';
 
 /**
  * Normalize data from OplogEntries for comparison in tests.
@@ -27,14 +33,14 @@ const normalizeOplogData = (data: OplogEntry['data']) => {
 export function registerDataStorageDataTests(generateStorageFactory: storage.TestStorageFactory) {
   test('removing row', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -59,7 +65,9 @@ bucket_definitions:
 
     const { checkpoint } = await bucketStorage.getCheckpoint();
 
-    const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
+    const batch = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
     const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
@@ -76,10 +84,12 @@ bucket_definitions:
       { op: 'REMOVE', object_id: 'test1', checksum: c2 }
     ]);
 
-    const checksums = [...(await bucketStorage.getChecksums(checkpoint, ['global[]'])).values()];
+    const checksums = [
+      ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
+    ];
     expect(checksums).toEqual([
       {
-        bucket: 'global[]',
+        bucket: bucketRequest(syncRules, 'global[]'),
         checksum: (c1 + c2) & 0xffffffff,
         count: 2
       }
@@ -88,14 +98,14 @@ bucket_definitions:
 
   test('changing client ids', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT client_id as id, description FROM "%"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = TEST_TABLE;
@@ -135,7 +145,9 @@ bucket_definitions:
       await batch.commit('1/1');
     });
     const { checkpoint } = await bucketStorage.getCheckpoint();
-    const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
+    const batch = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
     const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
@@ -153,14 +165,14 @@ bucket_definitions:
 
   test('re-apply delete', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -201,7 +213,9 @@ bucket_definitions:
 
     const { checkpoint } = await bucketStorage.getCheckpoint();
 
-    const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
+    const batch = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
     const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
@@ -218,10 +232,12 @@ bucket_definitions:
       { op: 'REMOVE', object_id: 'test1', checksum: c2 }
     ]);
 
-    const checksums = [...(await bucketStorage.getChecksums(checkpoint, ['global[]'])).values()];
+    const checksums = [
+      ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
+    ];
     expect(checksums).toEqual([
       {
-        bucket: 'global[]',
+        bucket: bucketRequest(syncRules, 'global[]'),
         checksum: (c1 + c2) & 0xffffffff,
         count: 2
       }
@@ -230,14 +246,14 @@ bucket_definitions:
 
   test('re-apply update + delete', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -320,7 +336,9 @@ bucket_definitions:
 
     const { checkpoint } = await bucketStorage.getCheckpoint();
 
-    const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
+    const batch = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
 
     const data = batch[0].chunkData.data.map((d) => {
       return {
@@ -340,10 +358,12 @@ bucket_definitions:
       { op: 'REMOVE', object_id: 'test1', checksum: c2 }
     ]);
 
-    const checksums = [...(await bucketStorage.getChecksums(checkpoint, ['global[]'])).values()];
+    const checksums = [
+      ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
+    ];
     expect(checksums).toEqual([
       {
-        bucket: 'global[]',
+        bucket: bucketRequest(syncRules, 'global[]'),
         checksum: (c1 + c1 + c1 + c2) & 0xffffffff,
         count: 4
       }
@@ -360,14 +380,14 @@ bucket_definitions:
     // 2. Output order not being correct.
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     // Pre-setup
@@ -480,7 +500,7 @@ bucket_definitions:
     const checkpoint2 = result2!.flushed_op;
 
     const batch = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', checkpoint1]]))
+      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', checkpoint1]]))
     );
 
     const data = batch[0].chunkData.data.map((d) => {
@@ -518,14 +538,14 @@ bucket_definitions:
       ]);
     }
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
@@ -580,7 +600,7 @@ bucket_definitions:
     const checkpoint3 = result3!.flushed_op;
 
     const batch = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]]))
+      bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]]))
     );
     const data = batch[0].chunkData.data.map((d) => {
       return {
@@ -626,14 +646,14 @@ bucket_definitions:
     }
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
@@ -688,7 +708,7 @@ bucket_definitions:
     const checkpoint3 = result3!.flushed_op;
 
     const batch = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]]))
+      bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]]))
     );
     const data = batch[0].chunkData.data.map((d) => {
       return {
@@ -724,14 +744,14 @@ bucket_definitions:
     // The specific batch splits is an implementation detail of the storage driver,
     // and the test will have to updated when other implementations are added.
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -790,7 +810,7 @@ bucket_definitions:
     };
 
     const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
     );
     expect(test_utils.getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -805,7 +825,7 @@ bucket_definitions:
     const batch2 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([['global[]', BigInt(batch1[0].chunkData.next_after)]]),
+        bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
         options
       )
     );
@@ -822,7 +842,7 @@ bucket_definitions:
     const batch3 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([['global[]', BigInt(batch2[0].chunkData.next_after)]]),
+        bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
         options
       )
     );
@@ -833,14 +853,14 @@ bucket_definitions:
   test('long batch', async () => {
     // Test syncing a batch of data that is limited by count.
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "%"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -864,7 +884,7 @@ bucket_definitions:
     const { checkpoint } = await bucketStorage.getCheckpoint();
 
     const batch1 = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), { limit: 4 })
+      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), { limit: 4 })
     );
 
     expect(test_utils.getBatchData(batch1)).toEqual([
@@ -881,9 +901,13 @@ bucket_definitions:
     });
 
     const batch2 = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch1.chunkData.next_after)]]), {
-        limit: 4
-      })
+      bucketStorage.getBucketDataBatch(
+        checkpoint,
+        bucketRequestMap(syncRules, [['global[]', BigInt(batch1.chunkData.next_after)]]),
+        {
+          limit: 4
+        }
+      )
     );
     expect(test_utils.getBatchData(batch2)).toEqual([
       { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
@@ -897,9 +921,13 @@ bucket_definitions:
     });
 
     const batch3 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch2.chunkData.next_after)]]), {
-        limit: 4
-      })
+      bucketStorage.getBucketDataBatch(
+        checkpoint,
+        bucketRequestMap(syncRules, [['global[]', BigInt(batch2.chunkData.next_after)]]),
+        {
+          limit: 4
+        }
+      )
     );
     expect(test_utils.getBatchData(batch3)).toEqual([]);
 
@@ -909,8 +937,8 @@ bucket_definitions:
   describe('batch has_more', () => {
     const setup = async (options: BucketDataBatchOptions) => {
       await using factory = await generateStorageFactory();
-      const syncRules = await factory.updateSyncRules({
-        content: `
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
   bucket_definitions:
     global1:
       data:
@@ -918,8 +946,8 @@ bucket_definitions:
     global2:
       data:
         - SELECT id, description FROM test WHERE bucket = 'global2'
-  `
-      });
+  `)
+      );
       const bucketStorage = factory.getInstance(syncRules);
 
       await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -942,24 +970,26 @@ bucket_definitions:
       });
 
       const { checkpoint } = await bucketStorage.getCheckpoint();
-      return await test_utils.fromAsync(
+      const batch = await test_utils.fromAsync(
         bucketStorage.getBucketDataBatch(
           checkpoint,
-          new Map([
+          bucketRequestMap(syncRules, [
             ['global1[]', 0n],
             ['global2[]', 0n]
           ]),
           options
         )
       );
+
+      return { syncRules, batch };
     };
 
     test('batch has_more (1)', async () => {
-      const batch = await setup({ limit: 5 });
+      const { batch, syncRules } = await setup({ limit: 5 });
       expect(batch.length).toEqual(2);
 
-      expect(batch[0].chunkData.bucket).toEqual('global1[]');
-      expect(batch[1].chunkData.bucket).toEqual('global2[]');
+      expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
+      expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
 
       expect(test_utils.getBatchData(batch[0])).toEqual([
         { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -986,11 +1016,11 @@ bucket_definitions:
     });
 
     test('batch has_more (2)', async () => {
-      const batch = await setup({ limit: 11 });
+      const { batch, syncRules } = await setup({ limit: 11 });
       expect(batch.length).toEqual(2);
 
-      expect(batch[0].chunkData.bucket).toEqual('global1[]');
-      expect(batch[1].chunkData.bucket).toEqual('global2[]');
+      expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
+      expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
 
       expect(test_utils.getBatchData(batch[0])).toEqual([
         { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -1023,12 +1053,12 @@ bucket_definitions:
 
     test('batch has_more (3)', async () => {
       // 50 bytes is more than 1 row, less than 2 rows
-      const batch = await setup({ limit: 3, chunkLimitBytes: 50 });
+      const { batch, syncRules } = await setup({ limit: 3, chunkLimitBytes: 50 });
 
       expect(batch.length).toEqual(3);
-      expect(batch[0].chunkData.bucket).toEqual('global1[]');
-      expect(batch[1].chunkData.bucket).toEqual('global2[]');
-      expect(batch[2].chunkData.bucket).toEqual('global2[]');
+      expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
+      expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
+      expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
 
       expect(test_utils.getBatchData(batch[0])).toEqual([
         { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -1070,14 +1100,14 @@ bucket_definitions:
       replication_size_bytes: 0
     });
 
-    const r = await f.configureSyncRules({ content: 'bucket_definitions: {}', validate: false });
+    const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
     const storage = f.getInstance(r.persisted_sync_rules!);
     await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.keepalive('1/0');
     });
 
-    const metrics2 = await f.getStorageMetrics();
-    expect(metrics2).toMatchSnapshot();
+    await f.getStorageMetrics();
+    // We don't care about the specific values here
   });
 
   test('op_id initialization edge case', async () => {
@@ -1085,15 +1115,15 @@ bucket_definitions:
     // but large enough in size to be split over multiple returned chunks.
     // Similar to the above test, but splits over 1MB chunks.
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
   bucket_definitions:
     global:
       data:
         - SELECT id FROM test
         - SELECT id FROM test_ignore WHERE false
-  `
-    });
+  `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = test_utils.makeTestTable('test', ['id']);
@@ -1132,14 +1162,14 @@ bucket_definitions:
 
   test('unchanged checksums', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   global:
     data:
       - SELECT client_id as id, description FROM "%"
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = TEST_TABLE;
@@ -1157,10 +1187,14 @@ bucket_definitions:
     });
     const { checkpoint } = await bucketStorage.getCheckpoint();
 
-    const checksums = [...(await bucketStorage.getChecksums(checkpoint, ['global[]'])).values()];
-    expect(checksums).toEqual([{ bucket: 'global[]', checksum: 1917136889, count: 1 }]);
-    const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, ['global[]'])).values()];
-    expect(checksums2).toEqual([{ bucket: 'global[]', checksum: 1917136889, count: 1 }]);
+    const checksums = [
+      ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
+    ];
+    expect(checksums).toEqual([{ bucket: bucketRequest(syncRules, 'global[]'), checksum: 1917136889, count: 1 }]);
+    const checksums2 = [
+      ...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values()
+    ];
+    expect(checksums2).toEqual([{ bucket: bucketRequest(syncRules, 'global[]'), checksum: 1917136889, count: 1 }]);
   });
 
   testChecksumBatching(generateStorageFactory);
@@ -1174,15 +1208,15 @@ bucket_definitions:
 export function testChecksumBatching(generateStorageFactory: storage.TestStorageFactory) {
   test('checksums for multiple buckets', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   user:
     parameters: select request.user_id() as user_id
     data:
       - select id, description from test where user_id = bucket.user_id
-`
-    });
+`)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const sourceTable = TEST_TABLE;
@@ -1207,14 +1241,14 @@ bucket_definitions:
     const { checkpoint } = await bucketStorage.getCheckpoint();
 
     bucketStorage.clearChecksumCache();
-    const buckets = ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]'];
+    const buckets = bucketRequests(syncRules, ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]']);
     const checksums = [...(await bucketStorage.getChecksums(checkpoint, buckets)).values()];
     checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
     expect(checksums).toEqual([
-      { bucket: 'user["u1"]', count: 4, checksum: 346204588 },
-      { bucket: 'user["u2"]', count: 4, checksum: 5261081 },
-      { bucket: 'user["u3"]', count: 4, checksum: 134760718 },
-      { bucket: 'user["u4"]', count: 4, checksum: -302639724 }
+      { bucket: bucketRequest(syncRules, 'user["u1"]'), count: 4, checksum: 346204588 },
+      { bucket: bucketRequest(syncRules, 'user["u2"]'), count: 4, checksum: 5261081 },
+      { bucket: bucketRequest(syncRules, 'user["u3"]'), count: 4, checksum: 134760718 },
+      { bucket: bucketRequest(syncRules, 'user["u4"]'), count: 4, checksum: -302639724 }
     ]);
   });
 }