@powersync/service-core-tests 0.9.3 → 0.9.5

package/dist/tests/register-sync-tests.js

@@ -207,8 +207,7 @@ bucket_definitions:
                 await result_2;
         }
     });
-    // Temporarily skipped - interruption disabled
-    test.skip('sync interrupts low-priority buckets on new checkpoints', async () => {
+    test('sync interrupts low-priority buckets on new checkpoints', async () => {
         const env_3 = { stack: [], error: void 0, hasError: false };
         try {
             const f = __addDisposableResource(env_3, await factory(), true);
@@ -319,10 +318,290 @@ bucket_definitions:
                 await result_3;
         }
     });
-    test('sends checkpoint complete line for empty checkpoint', async () => {
+    test('sync interruptions with unrelated data', async () => {
         const env_4 = { stack: [], error: void 0, hasError: false };
         try {
             const f = __addDisposableResource(env_4, await factory(), true);
+            const syncRules = await f.updateSyncRules({
+                content: `
+bucket_definitions:
+  b0:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b1:
+    priority: 1
+    parameters: SELECT request.user_id() as user_id
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 5 AND description = bucket.user_id;
+    `
+            });
+            const bucketStorage = f.getInstance(syncRules);
+            await bucketStorage.autoActivate();
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                // Initial data: Add one priority row and 10k low-priority rows.
+                await batch.save({
+                    sourceTable: TEST_TABLE,
+                    tag: storage.SaveOperationTag.INSERT,
+                    after: {
+                        id: 'highprio',
+                        description: 'user_one'
+                    },
+                    afterReplicaId: 'highprio'
+                });
+                for (let i = 0; i < 10_000; i++) {
+                    await batch.save({
+                        sourceTable: TEST_TABLE,
+                        tag: storage.SaveOperationTag.INSERT,
+                        after: {
+                            id: `${i}`,
+                            description: 'low prio'
+                        },
+                        afterReplicaId: `${i}`
+                    });
+                }
+                await batch.commit('0/1');
+            });
+            const stream = sync.streamResponse({
+                syncContext,
+                bucketStorage,
+                syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+                params: {
+                    buckets: [],
+                    include_checksum: true,
+                    raw_data: true
+                },
+                tracker,
+                syncParams: new RequestParameters({ sub: 'user_one' }, {}),
+                token: { sub: 'user_one', exp: Date.now() / 1000 + 100000 }
+            });
+            let sentCheckpoints = 0;
+            let completedCheckpoints = 0;
+            let sentRows = 0;
+            // Expected flow:
+            //  1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
+            //  2. We insert a new row that is not part of a bucket relevant to this stream.
+            //  3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
+            //  4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
+            for await (let next of stream) {
+                if (typeof next == 'string') {
+                    next = JSON.parse(next);
+                }
+                if (typeof next === 'object' && next !== null) {
+                    if ('partial_checkpoint_complete' in next) {
+                        if (sentCheckpoints == 1) {
+                            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                                // Add a high-priority row that doesn't affect this sync stream.
+                                await batch.save({
+                                    sourceTable: TEST_TABLE,
+                                    tag: storage.SaveOperationTag.INSERT,
+                                    after: {
+                                        id: 'highprio2',
+                                        description: 'user_two'
+                                    },
+                                    afterReplicaId: 'highprio2'
+                                });
+                                await batch.commit('0/2');
+                            });
+                        }
+                        else {
+                            expect(sentCheckpoints).toBe(2);
+                            expect(sentRows).toBe(10002);
+                        }
+                    }
+                    if ('checkpoint' in next || 'checkpoint_diff' in next) {
+                        sentCheckpoints += 1;
+                    }
+                    if ('data' in next) {
+                        sentRows += next.data.data.length;
+                    }
+                    if ('checkpoint_complete' in next) {
+                        completedCheckpoints++;
+                        if (completedCheckpoints == 2) {
+                            break;
+                        }
+                        if (completedCheckpoints == 1) {
+                            expect(sentRows).toBe(10001);
+                            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                                // Add a high-priority row that affects this sync stream.
+                                await batch.save({
+                                    sourceTable: TEST_TABLE,
+                                    tag: storage.SaveOperationTag.INSERT,
+                                    after: {
+                                        id: 'highprio3',
+                                        description: 'user_one'
+                                    },
+                                    afterReplicaId: 'highprio3'
+                                });
+                                await batch.commit('0/3');
+                            });
+                        }
+                    }
+                }
+            }
+            expect(sentCheckpoints).toBe(2);
+            expect(sentRows).toBe(10002);
+        }
+        catch (e_4) {
+            env_4.error = e_4;
+            env_4.hasError = true;
+        }
+        finally {
+            const result_4 = __disposeResources(env_4);
+            if (result_4)
+                await result_4;
+        }
+    });
+    test('sync interrupts low-priority buckets on new checkpoints (2)', async () => {
+        const env_5 = { stack: [], error: void 0, hasError: false };
+        try {
+            const f = __addDisposableResource(env_5, await factory(), true);
+            // bucket0a -> send all data
+            // then interrupt checkpoint with new data for all buckets
+            // -> data for all buckets should be sent in the new checkpoint
+            const syncRules = await f.updateSyncRules({
+                content: `
+bucket_definitions:
+  b0a:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b0b:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b1:
+    priority: 1
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 5;
+    `
+            });
+            const bucketStorage = f.getInstance(syncRules);
+            await bucketStorage.autoActivate();
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                // Initial data: Add one priority row and 10k low-priority rows.
+                await batch.save({
+                    sourceTable: TEST_TABLE,
+                    tag: storage.SaveOperationTag.INSERT,
+                    after: {
+                        id: 'highprio',
+                        description: 'High priority row'
+                    },
+                    afterReplicaId: 'highprio'
+                });
+                for (let i = 0; i < 2_000; i++) {
+                    await batch.save({
+                        sourceTable: TEST_TABLE,
+                        tag: storage.SaveOperationTag.INSERT,
+                        after: {
+                            id: `${i}`,
+                            description: 'low prio'
+                        },
+                        afterReplicaId: `${i}`
+                    });
+                }
+                await batch.commit('0/1');
+            });
+            const stream = sync.streamResponse({
+                syncContext,
+                bucketStorage,
+                syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+                params: {
+                    buckets: [],
+                    include_checksum: true,
+                    raw_data: true
+                },
+                tracker,
+                syncParams: new RequestParameters({ sub: '' }, {}),
+                token: { exp: Date.now() / 1000 + 10 }
+            });
+            let sentRows = 0;
+            let lines = [];
+            for await (let next of stream) {
+                if (typeof next == 'string') {
+                    next = JSON.parse(next);
+                }
+                if (typeof next === 'object' && next !== null) {
+                    if ('partial_checkpoint_complete' in next) {
+                        lines.push(next);
+                    }
+                    if ('checkpoint' in next || 'checkpoint_diff' in next) {
+                        lines.push(next);
+                    }
+                    if ('data' in next) {
+                        lines.push({ data: { ...next.data, data: undefined } });
+                        sentRows += next.data.data.length;
+                        if (sentRows == 1001) {
+                            // Save new data to interrupt the low-priority sync.
+                            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+                                // Add another high-priority row. This should interrupt the long-running low-priority sync.
+                                await batch.save({
+                                    sourceTable: TEST_TABLE,
+                                    tag: storage.SaveOperationTag.INSERT,
+                                    after: {
+                                        id: 'highprio2',
+                                        description: 'Another high-priority row'
+                                    },
+                                    afterReplicaId: 'highprio2'
+                                });
+                                // Also add a low-priority row
+                                await batch.save({
+                                    sourceTable: TEST_TABLE,
+                                    tag: storage.SaveOperationTag.INSERT,
+                                    after: {
+                                        id: '2001',
+                                        description: 'Another low-priority row'
+                                    },
+                                    afterReplicaId: '2001'
+                                });
+                                await batch.commit('0/2');
+                            });
+                            if (sentRows >= 1000 && sentRows <= 2001) {
+                                // pause for a bit to give the stream time to process interruptions.
+                                // This covers the data batch above and the next one.
+                                await timers.setTimeout(50);
+                            }
+                        }
+                    }
+                    if ('checkpoint_complete' in next) {
+                        lines.push(next);
+                        break;
+                    }
+                }
+            }
+            // Expected lines (full details in snapshot):
+            //
+            // checkpoint (4001)
+            // data (b1[] 0 -> 1)
+            // partial_checkpoint_complete (4001, priority 1)
+            // data (b0a[], 0 -> 2000)
+            // ## adds new data, interrupting the checkpoint
+            // data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
+            // checkpoint_diff (4004)
+            // data (b1[], 1 -> 4002)
+            // partial_checkpoint_complete (4004, priority 1)
+            // data (b0a[], 4000 -> 4003)
+            // data (b0b[], 0 -> 1999)
+            // data (b0b[], 1999 -> 3999)
+            // data (b0b[], 3999 -> 4004)
+            // checkpoint_complete (4004)
+            expect(lines).toMatchSnapshot();
+            expect(sentRows).toBe(4004);
+        }
+        catch (e_5) {
+            env_5.error = e_5;
+            env_5.hasError = true;
+        }
+        finally {
+            const result_5 = __disposeResources(env_5);
+            if (result_5)
+                await result_5;
+        }
+    });
+    test('sends checkpoint complete line for empty checkpoint', async () => {
+        const env_6 = { stack: [], error: void 0, hasError: false };
+        try {
+            const f = __addDisposableResource(env_6, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: BASIC_SYNC_RULES
             });
@@ -378,14 +657,14 @@ bucket_definitions:
             }
             expect(lines).toMatchSnapshot();
         }
-        catch (e_4) {
-            env_4.error = e_4;
-            env_4.hasError = true;
+        catch (e_6) {
+            env_6.error = e_6;
+            env_6.hasError = true;
         }
         finally {
-            const result_4 = __disposeResources(env_4);
-            if (result_4)
-                await result_4;
+            const result_6 = __disposeResources(env_6);
+            if (result_6)
+                await result_6;
         }
     });
     test('sync legacy non-raw data', async () => {
@@ -427,9 +706,9 @@ bucket_definitions:
         expect(lines[1].data.data[0].data.large_num).toEqual(12345678901234567890n);
     });
     test('expired token', async () => {
-        const env_5 = { stack: [], error: void 0, hasError: false };
+        const env_7 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_5, await factory(), true);
+            const f = __addDisposableResource(env_7, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: BASIC_SYNC_RULES
             });
@@ -451,20 +730,20 @@ bucket_definitions:
             const lines = await consumeCheckpointLines(stream);
             expect(lines).toMatchSnapshot();
         }
-        catch (e_5) {
-            env_5.error = e_5;
-            env_5.hasError = true;
+        catch (e_7) {
+            env_7.error = e_7;
+            env_7.hasError = true;
         }
         finally {
-            const result_5 = __disposeResources(env_5);
-            if (result_5)
-                await result_5;
+            const result_7 = __disposeResources(env_7);
+            if (result_7)
+                await result_7;
         }
     });
     test('sync updates to global data', async (context) => {
-        const env_6 = { stack: [], error: void 0, hasError: false };
+        const env_8 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_6, await factory(), true);
+            const f = __addDisposableResource(env_8, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: BASIC_SYNC_RULES
             });
@@ -515,20 +794,20 @@ bucket_definitions:
             });
             expect(await getCheckpointLines(iter)).toMatchSnapshot();
         }
-        catch (e_6) {
-            env_6.error = e_6;
-            env_6.hasError = true;
+        catch (e_8) {
+            env_8.error = e_8;
+            env_8.hasError = true;
         }
         finally {
-            const result_6 = __disposeResources(env_6);
-            if (result_6)
-                await result_6;
+            const result_8 = __disposeResources(env_8);
+            if (result_8)
+                await result_8;
         }
     });
     test('sync updates to parameter query only', async (context) => {
-        const env_7 = { stack: [], error: void 0, hasError: false };
+        const env_9 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_7, await factory(), true);
+            const f = __addDisposableResource(env_9, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: `bucket_definitions:
   by_user:
@@ -579,20 +858,20 @@ bucket_definitions:
             expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual(['by_user["user1"]']);
             expect(checkpoint2).toMatchSnapshot();
         }
-        catch (e_7) {
-            env_7.error = e_7;
-            env_7.hasError = true;
+        catch (e_9) {
+            env_9.error = e_9;
+            env_9.hasError = true;
         }
         finally {
-            const result_7 = __disposeResources(env_7);
-            if (result_7)
-                await result_7;
+            const result_9 = __disposeResources(env_9);
+            if (result_9)
+                await result_9;
         }
     });
     test('sync updates to data query only', async (context) => {
-        const env_8 = { stack: [], error: void 0, hasError: false };
+        const env_10 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_8, await factory(), true);
+            const f = __addDisposableResource(env_10, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: `bucket_definitions:
   by_user:
@@ -656,20 +935,20 @@ bucket_definitions:
             expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual(['by_user["user1"]']);
             expect(checkpoint2).toMatchSnapshot();
         }
-        catch (e_8) {
-            env_8.error = e_8;
-            env_8.hasError = true;
+        catch (e_10) {
+            env_10.error = e_10;
+            env_10.hasError = true;
         }
         finally {
-            const result_8 = __disposeResources(env_8);
-            if (result_8)
-                await result_8;
+            const result_10 = __disposeResources(env_10);
+            if (result_10)
+                await result_10;
         }
     });
     test('sync updates to parameter query + data', async (context) => {
-        const env_9 = { stack: [], error: void 0, hasError: false };
+        const env_11 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_9, await factory(), true);
+            const f = __addDisposableResource(env_11, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: `bucket_definitions:
   by_user:
@@ -727,20 +1006,20 @@ bucket_definitions:
             expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual(['by_user["user1"]']);
             expect(checkpoint2).toMatchSnapshot();
         }
-        catch (e_9) {
-            env_9.error = e_9;
-            env_9.hasError = true;
+        catch (e_11) {
+            env_11.error = e_11;
+            env_11.hasError = true;
         }
         finally {
-            const result_9 = __disposeResources(env_9);
-            if (result_9)
-                await result_9;
+            const result_11 = __disposeResources(env_11);
+            if (result_11)
+                await result_11;
         }
     });
     test('expiring token', async (context) => {
-        const env_10 = { stack: [], error: void 0, hasError: false };
+        const env_12 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_10, await factory(), true);
+            const f = __addDisposableResource(env_12, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: BASIC_SYNC_RULES
             });
@@ -769,24 +1048,24 @@ bucket_definitions:
             const expLines = await getCheckpointLines(iter);
             expect(expLines).toMatchSnapshot();
         }
-        catch (e_10) {
-            env_10.error = e_10;
-            env_10.hasError = true;
+        catch (e_12) {
+            env_12.error = e_12;
+            env_12.hasError = true;
         }
         finally {
-            const result_10 = __disposeResources(env_10);
-            if (result_10)
-                await result_10;
+            const result_12 = __disposeResources(env_12);
+            if (result_12)
+                await result_12;
         }
     });
     test('compacting data - invalidate checkpoint', async (context) => {
-        const env_11 = { stack: [], error: void 0, hasError: false };
+        const env_13 = { stack: [], error: void 0, hasError: false };
         try {
             // This tests a case of a compact operation invalidating a checkpoint in the
             // middle of syncing data.
             // This is expected to be rare in practice, but it is important to handle
             // this case correctly to maintain consistency on the client.
-            const f = __addDisposableResource(env_11, await factory(), true);
+            const f = __addDisposableResource(env_13, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: BASIC_SYNC_RULES
             });
@@ -905,20 +1184,20 @@ bucket_definitions:
                 })
             });
         }
-        catch (e_11) {
-            env_11.error = e_11;
-            env_11.hasError = true;
+        catch (e_13) {
+            env_13.error = e_13;
+            env_13.hasError = true;
         }
         finally {
-            const result_11 = __disposeResources(env_11);
-            if (result_11)
-                await result_11;
+            const result_13 = __disposeResources(env_13);
+            if (result_13)
+                await result_13;
         }
     });
     test('write checkpoint', async () => {
-        const env_12 = { stack: [], error: void 0, hasError: false };
+        const env_14 = { stack: [], error: void 0, hasError: false };
         try {
-            const f = __addDisposableResource(env_12, await factory(), true);
+            const f = __addDisposableResource(env_14, await factory(), true);
             const syncRules = await f.updateSyncRules({
                 content: BASIC_SYNC_RULES
             });
@@ -970,14 +1249,14 @@ bucket_definitions:
                 })
             });
         }
-        catch (e_12) {
-            env_12.error = e_12;
-            env_12.hasError = true;
+        catch (e_14) {
+            env_14.error = e_14;
+            env_14.hasError = true;
         }
         finally {
-            const result_12 = __disposeResources(env_12);
-            if (result_12)
-                await result_12;
+            const result_14 = __disposeResources(env_14);
+            if (result_14)
+                await result_14;
         }
     });
 }