@powersync/service-core-tests 0.15.5 → 0.17.0

package/src/tests/register-data-storage-data-tests.ts

@@ -38,7 +38,8 @@ export function registerDataStorageDataTests(config: storage.TestStorageConfig)
 
   test('removing row', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -49,7 +50,7 @@ bucket_definitions:
         { storageVersion }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
 
@@ -103,7 +104,8 @@ bucket_definitions:
 
   test('insert after delete in new batch', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -114,7 +116,7 @@ bucket_definitions:
         { storageVersion }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -167,7 +169,8 @@ bucket_definitions:
   test('update after delete in new batch', async () => {
     // Update after delete may not be common, but the storage layer should handle it in an eventually-consistent way.
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -178,7 +181,7 @@ bucket_definitions:
         { storageVersion }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -234,7 +237,8 @@ bucket_definitions:
 
   test('insert after delete in same batch', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -247,7 +251,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -296,7 +300,8 @@ bucket_definitions:
 
   test('(insert, delete, insert), (delete)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -309,7 +314,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     {
       await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
       const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -374,7 +379,8 @@ bucket_definitions:
 
   test('changing client ids', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -387,7 +393,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -445,7 +451,8 @@ bucket_definitions:
 
   test('re-apply delete', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -458,7 +465,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -521,7 +528,8 @@ bucket_definitions:
 
   test('re-apply update + delete', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -532,7 +540,7 @@ bucket_definitions:
         { storageVersion }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -651,7 +659,8 @@ bucket_definitions:
     // 2. Output order not being correct.
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -662,7 +671,7 @@ bucket_definitions:
         { storageVersion }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
 
@@ -810,7 +819,8 @@ bucket_definitions:
       ]);
     }
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -823,10 +833,10 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-    const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config);
 
     // Pre-setup
     await writer.markAllSnapshotDone('1/1');
@@ -923,7 +933,8 @@ bucket_definitions:
     }
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -936,10 +947,10 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-    const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config);
 
     // Pre-setup
     await writer.markAllSnapshotDone('1/1');
@@ -1026,7 +1037,8 @@ bucket_definitions:
     // The specific batch splits is an implementation detail of the storage driver,
     // and the test will have to updated when other implementations are added.
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1039,7 +1051,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -1138,7 +1150,8 @@ bucket_definitions:
   test('long batch', async () => {
     // Test syncing a batch of data that is limited by count.
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1151,7 +1164,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -1217,7 +1230,8 @@ bucket_definitions:
   describe('batch has_more', () => {
     const setup = async (options: BucketDataBatchOptions) => {
       await using factory = await generateStorageFactory();
-      const syncRules = await factory.updateSyncRules(
+      const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+        factory,
         updateSyncRulesFromYaml(
           `
   bucket_definitions:
@@ -1231,7 +1245,7 @@ bucket_definitions:
           { storageVersion }
         )
       );
-      const bucketStorage = factory.getInstance(syncRules);
+      const bucketStorage = factory.getInstance(replicationStream);
       await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
       const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
       await writer.markAllSnapshotDone('1/1');
@@ -1332,39 +1346,67 @@ bucket_definitions:
       // 50 bytes is more than 1 row, less than 2 rows
       const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
 
-      expect(batch.length).toEqual(3);
-      expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
-      expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
-      expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
-
-      expect(test_utils.getBatchData(batch[0])).toEqual([
-        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
-      ]);
-
-      expect(test_utils.getBatchData(batch[1])).toEqual([
-        { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 }
-      ]);
-      expect(test_utils.getBatchData(batch[2])).toEqual([
-        { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
-      ]);
+      if (config.compressedBucketStorage) {
+        // In v3+, ops in the same bucket share a document, so ops 2 and 3 (global2) are batched together
+        expect(batch.length).toEqual(2);
+        expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
+        expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
+
+        expect(test_utils.getBatchData(batch[0])).toEqual([
+          { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+        ]);
+
+        expect(test_utils.getBatchData(batch[1])).toEqual([
+          { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
+          { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+        ]);
+
+        expect(test_utils.getBatchMeta(batch[0])).toEqual({
+          after: '0',
+          has_more: false,
+          next_after: '1'
+        });
 
-      expect(test_utils.getBatchMeta(batch[0])).toEqual({
-        after: '0',
-        has_more: false,
-        next_after: '1'
-      });
+        expect(test_utils.getBatchMeta(batch[1])).toEqual({
+          after: '0',
+          has_more: true,
+          next_after: '3'
+        });
+      } else {
+        expect(batch.length).toEqual(3);
+        expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
+        expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
+        expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
+
+        expect(test_utils.getBatchData(batch[0])).toEqual([
+          { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+        ]);
+
+        expect(test_utils.getBatchData(batch[1])).toEqual([
+          { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 }
+        ]);
+        expect(test_utils.getBatchData(batch[2])).toEqual([
+          { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+        ]);
+
+        expect(test_utils.getBatchMeta(batch[0])).toEqual({
+          after: '0',
+          has_more: false,
+          next_after: '1'
+        });
 
-      expect(test_utils.getBatchMeta(batch[1])).toEqual({
-        after: '0',
-        has_more: true,
-        next_after: '2'
-      });
+        expect(test_utils.getBatchMeta(batch[1])).toEqual({
+          after: '0',
+          has_more: true,
+          next_after: '2'
+        });
 
-      expect(test_utils.getBatchMeta(batch[2])).toEqual({
-        after: '2',
-        has_more: true,
-        next_after: '3'
-      });
+        expect(test_utils.getBatchMeta(batch[2])).toEqual({
+          after: '2',
+          has_more: true,
+          next_after: '3'
+        });
+      }
     });
   });
 
@@ -1392,7 +1434,8 @@ bucket_definitions:
     // but large enough in size to be split over multiple returned chunks.
     // Similar to the above test, but splits over 1MB chunks.
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
   bucket_definitions:
@@ -1406,7 +1449,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
 
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
@@ -1444,7 +1487,8 @@ bucket_definitions:
 
   test('unchanged checksums', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1457,7 +1501,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
 
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -1485,7 +1529,8 @@ bucket_definitions:
 
   test('empty checkpoints (1)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1496,7 +1541,7 @@ bucket_definitions:
         { storageVersion }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await writer.markAllSnapshotDone('1/1');
     await writer.commit('1/1');
@@ -1521,7 +1566,8 @@ bucket_definitions:
 
   test('empty checkpoints (2)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1534,7 +1580,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
@@ -1571,7 +1617,8 @@ bucket_definitions:
 
   test('empty checkpoints (sync rule activation)', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1584,7 +1631,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
@@ -1609,8 +1656,8 @@ bucket_definitions:
     const cp2 = await bucketStorage.getCheckpoint();
     expect(cp2.lsn).toEqual('3/1');
 
-    const activeSyncRules = await factory.getActiveSyncRulesContent();
-    expect(activeSyncRules?.id).toEqual(syncRules.id);
+    const activeSyncRules = await factory.getActiveSyncConfig();
+    expect(activeSyncRules?.content.replicationStreamId).toEqual(syncRules.replicationStreamId);
 
     // At this point, it should be a truely empty checkpoint
     const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
@@ -1623,7 +1670,8 @@ bucket_definitions:
 
   test.runIf(storageVersion >= 3)('deleting while streaming', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1636,7 +1684,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using snapshotWriter = await bucketStorage.createWriter({
       ...test_utils.BATCH_OPTIONS,
       skipExistingRows: true
@@ -1692,7 +1740,8 @@ export function testChecksumBatching(config: storage.TestStorageConfig) {
   const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
   test('checksums for multiple buckets', async () => {
     await using factory = await config.factory();
-    const syncRules = await factory.updateSyncRules(
+    const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
+      factory,
       updateSyncRulesFromYaml(
         `
 bucket_definitions:
@@ -1706,7 +1755,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = factory.getInstance(replicationStream);
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
     await writer.markAllSnapshotDone('1/1');
@@ -1756,7 +1805,11 @@ streams:
 `)
     );
 
-    const { errors } = deployed.parsed({ defaultSchema: 'ignored' }).sync_rules;
+    const [deployedContent] = deployed.syncConfigContent;
+    expect(deployedContent).toBeDefined();
+    const [deployedConfig] = deployedContent.parsed({ defaultSchema: 'ignored' }).syncConfigs;
+    expect(deployedConfig).toBeDefined();
+    const { errors } = deployedConfig;
     expect(errors).toHaveLength(1);
     expect(errors[0].message).toStrictEqual('Expected a SELECT statement');
     expect(errors[0].location).toStrictEqual({