@powersync/service-core-tests 0.9.4 → 0.10.0

package/src/tests/register-data-storage-tests.ts

@@ -404,7 +404,7 @@ bucket_definitions:
 
     const parameters = new RequestParameters({ sub: 'u1' }, {});
 
-    const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
+    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
 
     const lookups = q1.getLookups(parameters);
     expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]);
@@ -474,7 +474,7 @@ bucket_definitions:
 
     const parameters = new RequestParameters({ sub: 'unknown' }, {});
 
-    const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
+    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
 
     const lookups = q1.getLookups(parameters);
     expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]);
@@ -564,7 +564,7 @@ bucket_definitions:
     const parameters = new RequestParameters({ sub: 'u1' }, {});
 
     // Test intermediate values - could be moved to sync_rules.test.ts
-    const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
+    const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0];
     const lookups1 = q1.getLookups(parameters);
     expect(lookups1).toEqual([ParameterLookup.normalized('by_workspace', '1', [])]);
 
@@ -572,7 +572,7 @@ bucket_definitions:
     parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const q2 = sync_rules.bucket_descriptors[0].parameter_queries[1];
+    const q2 = sync_rules.bucketDescriptors[0].parameterQueries[1];
     const lookups2 = q2.getLookups(parameters);
     expect(lookups2).toEqual([ParameterLookup.normalized('by_workspace', '2', ['u1'])]);
 
@@ -1769,14 +1769,56 @@ bucket_definitions:
       .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
       [Symbol.asyncIterator]();
 
-    await bucketStorage.batchCreateCustomWriteCheckpoints([
-      {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.addCustomWriteCheckpoint({
         checkpoint: 5n,
         user_id: 'user1'
+      });
+      await batch.flush();
+      await batch.keepalive('5/0');
+    });
+
+    const result = await iter.next();
+    expect(result).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          lsn: '5/0'
+        },
+        writeCheckpoint: 5n
       }
-    ]);
+    });
+  });
+
+  test('custom write checkpoints - standalone checkpoint', async (context) => {
+    await using factory = await generateStorageFactory();
+    const r = await factory.configureSyncRules({
+      content: `
+bucket_definitions:
+  mybucket:
+    data: []
+    `,
+      validate: false
+    });
+    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    await bucketStorage.autoActivate();
+    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+
+    const abortController = new AbortController();
+    context.onTestFinished(() => abortController.abort());
+    const iter = bucketStorage
+      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      [Symbol.asyncIterator]();
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // Flush to clear state
+      await batch.flush();
+
+      await batch.addCustomWriteCheckpoint({
+        checkpoint: 5n,
+        user_id: 'user1'
+      });
+      await batch.flush();
       await batch.keepalive('5/0');
     });
 
@@ -1785,7 +1827,6 @@ bucket_definitions:
       done: false,
       value: {
         base: {
-          checkpoint: 0n,
           lsn: '5/0'
         },
         writeCheckpoint: 5n
@@ -1822,23 +1863,18 @@ bucket_definitions:
       done: false,
       value: {
         base: {
-          checkpoint: 0n,
           lsn: '5/0'
         },
         writeCheckpoint: null
       }
     });
 
-    await bucketStorage.batchCreateCustomWriteCheckpoints([
-      {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      batch.addCustomWriteCheckpoint({
         checkpoint: 6n,
         user_id: 'user1'
-      }
-    ]);
-    // We have to trigger a new keepalive after the checkpoint, at least to cover postgres storage.
-    // This is what is effetively triggered with RouteAPI.createReplicationHead().
-    // MongoDB storage doesn't explicitly need this anymore.
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      });
+      await batch.flush();
       await batch.keepalive('6/0');
     });
 
@@ -1847,7 +1883,6 @@ bucket_definitions:
       done: false,
       value: {
         base: {
-          checkpoint: 0n
           // can be 5/0 or 6/0 - actual value not relevant for custom write checkpoints
           // lsn: '6/0'
         },
@@ -1855,13 +1890,12 @@ bucket_definitions:
       }
     });
 
-    await bucketStorage.batchCreateCustomWriteCheckpoints([
-      {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      batch.addCustomWriteCheckpoint({
         checkpoint: 7n,
         user_id: 'user1'
-      }
-    ]);
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      });
+      await batch.flush();
       await batch.keepalive('7/0');
     });
 
@@ -1870,7 +1904,6 @@ bucket_definitions:
       done: false,
       value: {
         base: {
-          checkpoint: 0n
           // can be 5/0, 6/0 or 7/0 - actual value not relevant for custom write checkpoints
           // lsn: '7/0'
         },

package/src/tests/register-sync-tests.ts

@@ -405,6 +405,155 @@ bucket_definitions:
     expect(sentRows).toBe(10002);
   });
 
+  test('sync interrupts low-priority buckets on new checkpoints (2)', async () => {
+    await using f = await factory();
+
+    // bucket0a -> send all data
+    // then interrupt checkpoint with new data for all buckets
+    // -> data for all buckets should be sent in the new checkpoint
+
+    const syncRules = await f.updateSyncRules({
+      content: `
+bucket_definitions:
+  b0a:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b0b:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b1:
+    priority: 1
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 5;
+    `
+    });
+
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // Initial data: Add one priority row and 10k low-priority rows.
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'highprio',
+          description: 'High priority row'
+        },
+        afterReplicaId: 'highprio'
+      });
+      for (let i = 0; i < 2_000; i++) {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: `${i}`,
+            description: 'low prio'
+          },
+          afterReplicaId: `${i}`
+        });
+      }
+
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      syncContext,
+      bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    let sentRows = 0;
+    let lines: any[] = [];
+
+    for await (let next of stream) {
+      if (typeof next == 'string') {
+        next = JSON.parse(next);
+      }
+      if (typeof next === 'object' && next !== null) {
+        if ('partial_checkpoint_complete' in next) {
+          lines.push(next);
+        }
+        if ('checkpoint' in next || 'checkpoint_diff' in next) {
+          lines.push(next);
+        }
+
+        if ('data' in next) {
+          lines.push({ data: { ...next.data, data: undefined } });
+          sentRows += next.data.data.length;
+
+          if (sentRows == 1001) {
+            // Save new data to interrupt the low-priority sync.
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+              // Add another high-priority row. This should interrupt the long-running low-priority sync.
+              await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: storage.SaveOperationTag.INSERT,
+                after: {
+                  id: 'highprio2',
+                  description: 'Another high-priority row'
+                },
+                afterReplicaId: 'highprio2'
+              });
+
+              // Also add a low-priority row
+              await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: storage.SaveOperationTag.INSERT,
+                after: {
+                  id: '2001',
+                  description: 'Another low-priority row'
+                },
+                afterReplicaId: '2001'
+              });
+
+              await batch.commit('0/2');
+            });
+          }
+
+          if (sentRows >= 1000 && sentRows <= 2001) {
+            // pause for a bit to give the stream time to process interruptions.
+            // This covers the data batch above and the next one.
+            await timers.setTimeout(50);
+          }
+        }
+        if ('checkpoint_complete' in next) {
+          lines.push(next);
+          break;
+        }
+      }
+    }
+
+    // Expected lines (full details in snapshot):
+    //
+    // checkpoint (4001)
+    // data (b1[] 0 -> 1)
+    // partial_checkpoint_complete (4001, priority 1)
+    // data (b0a[], 0 -> 2000)
+    // ## adds new data, interrupting the checkpoint
+    // data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
+    // checkpoint_diff (4004)
+    // data (b1[], 1 -> 4002)
+    // partial_checkpoint_complete (4004, priority 1)
+    // data (b0a[], 4000 -> 4003)
+    // data (b0b[], 0 -> 1999)
+    // data (b0b[], 1999 -> 3999)
+    // data (b0b[], 3999 -> 4004)
+    // checkpoint_complete (4004)
+    expect(lines).toMatchSnapshot();
+    expect(sentRows).toBe(4004);
+  });
+
   test('sends checkpoint complete line for empty checkpoint', async () => {
     await using f = await factory();