@powersync/service-core-tests 0.4.0 → 0.5.0

package/src/tests/register-data-storage-tests.ts

@@ -140,7 +140,7 @@ bucket_definitions:
     );
 
     await using factory = await generateStorageFactory();
-    await using bucketStorage = factory.getInstance(sync_rules);
+    const bucketStorage = factory.getInstance(sync_rules);
 
     const table = test_utils.makeTestTable('todos', ['id', 'list_id']);
 
@@ -394,13 +394,12 @@ bucket_definitions:
     const parameter_sets = await bucketStorage.getParameterSets(checkpoint, lookups);
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const buckets = await sync_rules.queryBucketIds({
+    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
       getParameterSets(lookups) {
         return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
+      }
     });
-    expect(buckets).toEqual(['by_workspace["workspace1"]']);
+    expect(buckets).toEqual([{ bucket: 'by_workspace["workspace1"]', priority: 3 }]);
   });
 
   test('save and load parameters with dynamic global buckets', async () => {
@@ -466,14 +465,16 @@ bucket_definitions:
     parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
 
-    const buckets = await sync_rules.queryBucketIds({
+    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
       getParameterSets(lookups) {
         return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
+      }
     });
-    buckets.sort();
-    expect(buckets).toEqual(['by_public_workspace["workspace1"]', 'by_public_workspace["workspace3"]']);
+    buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
+    expect(buckets).toEqual([
+      { bucket: 'by_public_workspace["workspace1"]', priority: 3 },
+      { bucket: 'by_public_workspace["workspace3"]', priority: 3 }
+    ]);
   });
 
   test('multiple parameter queries', async () => {
@@ -562,12 +563,13 @@ bucket_definitions:
     expect(parameter_sets2).toEqual([{ workspace_id: 'workspace3' }]);
 
     // Test final values - the important part
-    const buckets = await sync_rules.queryBucketIds({
-      getParameterSets(lookups) {
-        return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
-    });
+    const buckets = (
+      await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return bucketStorage.getParameterSets(checkpoint, lookups);
+        }
+      })
+    ).map((e) => e.bucket);
     buckets.sort();
     expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']);
   });
@@ -1415,84 +1417,6 @@ bucket_definitions:
     expect(test_utils.getBatchMeta(batch3)).toEqual(null);
   });
 
-  test('batch should be disposed automatically', async () => {
-    const sync_rules = test_utils.testRules(`
-      bucket_definitions:
-        global:
-          data: [] 
-          `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
-
-    let isDisposed = false;
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      batch.registerListener({
-        disposed: () => {
-          isDisposed = true;
-        }
-      });
-    });
-    expect(isDisposed).true;
-
-    isDisposed = false;
-    let errorCaught = false;
-    try {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        batch.registerListener({
-          disposed: () => {
-            isDisposed = true;
-          }
-        });
-        throw new Error(`Testing exceptions`);
-      });
-    } catch (ex) {
-      errorCaught = true;
-      expect(ex.message.includes('Testing')).true;
-    }
-    expect(errorCaught).true;
-    expect(isDisposed).true;
-  });
-
-  test('batch should be disposed automatically', async () => {
-    const sync_rules = test_utils.testRules(`
-      bucket_definitions:
-        global:
-          data: [] 
-          `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
-
-    let isDisposed = false;
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      batch.registerListener({
-        disposed: () => {
-          isDisposed = true;
-        }
-      });
-    });
-    expect(isDisposed).true;
-
-    isDisposed = false;
-    let errorCaught = false;
-    try {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        batch.registerListener({
-          disposed: () => {
-            isDisposed = true;
-          }
-        });
-        throw new Error(`Testing exceptions`);
-      });
-    } catch (ex) {
-      errorCaught = true;
-      expect(ex.message.includes('Testing')).true;
-    }
-    expect(errorCaught).true;
-    expect(isDisposed).true;
-  });
-
   test('empty storage metrics', async () => {
     await using f = await generateStorageFactory({ dropAll: true });
     const metrics = await f.getStorageMetrics();

package/src/tests/register-sync-tests.ts

@@ -67,13 +67,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -83,6 +83,176 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     expect(lines).toMatchSnapshot();
   });
 
+  test('sync buckets in order', async () => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `
+bucket_definitions:
+  b0:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 2;
+  b1:
+    priority: 1
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 2;
+    `
+    });
+
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1',
+          description: 'Test 1'
+        },
+        afterReplicaId: 't1'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'earlier',
+          description: 'Test 2'
+        },
+        afterReplicaId: 'earlier'
+      });
+
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    const lines = await consumeCheckpointLines(stream);
+    expect(lines).toMatchSnapshot();
+  });
+
+  test('sync interrupts low-priority buckets on new checkpoints', async () => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `
+bucket_definitions:
+  b0:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b1:
+    priority: 1
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 5;
+    `
+    });
+
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // Initial data: Add one priority row and 10k low-priority rows.
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'highprio',
+          description: 'High priority row'
+        },
+        afterReplicaId: 'highprio'
+      });
+      for (let i = 0; i < 10_000; i++) {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: `${i}`,
+            description: 'low prio'
+          },
+          afterReplicaId: `${i}`
+        });
+      }
+
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    let sentCheckpoints = 0;
+    let sentRows = 0;
+
+    for await (let next of stream) {
+      if (typeof next == 'string') {
+        next = JSON.parse(next);
+      }
+      if (typeof next === 'object' && next !== null) {
+        if ('partial_checkpoint_complete' in next) {
+          if (sentCheckpoints == 1) {
+            // Save new data to interrupt the low-priority sync.
+
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+              // Add another high-priority row. This should interrupt the long-running low-priority sync.
+              await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: storage.SaveOperationTag.INSERT,
+                after: {
+                  id: 'highprio2',
+                  description: 'Another high-priority row'
+                },
+                afterReplicaId: 'highprio2'
+              });
+
+              await batch.commit('0/2');
+            });
+          } else {
+            // Low-priority sync from the first checkpoint was interrupted. This should not happen before
+            // 1000 low-priority items were synchronized.
+            expect(sentCheckpoints).toBe(2);
+            expect(sentRows).toBeGreaterThan(1000);
+          }
+        }
+        if ('checkpoint' in next || 'checkpoint_diff' in next) {
+          sentCheckpoints += 1;
+        }
+
+        if ('data' in next) {
+          sentRows += next.data.data.length;
+        }
+        if ('checkpoint_complete' in next) {
+          break;
+        }
+      }
+    }
+
+    expect(sentCheckpoints).toBe(2);
+    expect(sentRows).toBe(10002);
+  });
+
   test('sync legacy non-raw data', async () => {
     const f = await factory();
 
@@ -109,13 +279,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: false
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -134,17 +304,17 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules);
-    await storage.autoActivate();
+    const bucketStorage = await f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
@@ -165,13 +335,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     await bucketStorage.autoActivate();
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -222,19 +392,19 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules);
-    await storage.autoActivate();
+    const bucketStorage = await f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
 
     const exp = Date.now() / 1000 + 0.1;
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
@@ -288,13 +458,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -411,13 +581,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const params: sync.SyncStreamParameters = {
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: 'test' }, {}),
       token: { sub: 'test', exp: Date.now() / 1000 + 10 } as any