@powersync/service-core-tests 0.9.2 → 0.9.4

package/src/tests/register-data-storage-tests.ts

@@ -1,6 +1,12 @@
-import { getUuidReplicaIdentityBson, OplogEntry, storage } from '@powersync/service-core';
+import {
+  BucketDataBatchOptions,
+  getUuidReplicaIdentityBson,
+  InternalOpId,
+  OplogEntry,
+  storage
+} from '@powersync/service-core';
 import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules';
-import { expect, test } from 'vitest';
+import { expect, test, describe, beforeEach } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
 
 export const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
@@ -338,7 +344,7 @@ bucket_definitions:
     const checkpoint = result!.flushed_op;
 
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
-    const data = batch[0].batch.data.map((d) => {
+    const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -635,7 +641,7 @@ bucket_definitions:
     });
     const checkpoint = result!.flushed_op;
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
-    const data = batch[0].batch.data.map((d) => {
+    const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id
@@ -699,7 +705,7 @@ bucket_definitions:
     const checkpoint = result!.flushed_op;
 
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
-    const data = batch[0].batch.data.map((d) => {
+    const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -815,7 +821,7 @@ bucket_definitions:
 
     const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]])));
 
-    const data = batch[0].batch.data.map((d) => {
+    const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -1014,7 +1020,7 @@ bucket_definitions:
       bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', checkpoint1]]))
     );
 
-    const data = batch[0].batch.data.map((d) => {
+    const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -1113,7 +1119,7 @@ bucket_definitions:
     const batch = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]]))
     );
-    const data = batch[0].batch.data.map((d) => {
+    const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -1221,7 +1227,7 @@ bucket_definitions:
     const batch = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]]))
     );
-    const data = batch[0].batch.data.map((d) => {
+    const data = batch[0].chunkData.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -1332,7 +1338,11 @@ bucket_definitions:
     });
 
     const batch2 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch1[0].batch.next_after)]]), options)
+      bucketStorage.getBucketDataBatch(
+        checkpoint,
+        new Map([['global[]', BigInt(batch1[0].chunkData.next_after)]]),
+        options
+      )
     );
     expect(test_utils.getBatchData(batch2)).toEqual([
       { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
@@ -1345,7 +1355,11 @@ bucket_definitions:
     });
 
     const batch3 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch2[0].batch.next_after)]]), options)
+      bucketStorage.getBucketDataBatch(
+        checkpoint,
+        new Map([['global[]', BigInt(batch2[0].chunkData.next_after)]]),
+        options
+      )
     );
     expect(test_utils.getBatchData(batch3)).toEqual([]);
     expect(test_utils.getBatchMeta(batch3)).toEqual(null);
@@ -1400,7 +1414,7 @@ bucket_definitions:
     });
 
     const batch2 = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch1.batch.next_after)]]), {
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch1.chunkData.next_after)]]), {
         limit: 4
       })
     );
@@ -1416,7 +1430,7 @@ bucket_definitions:
     });
 
     const batch3 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch2.batch.next_after)]]), {
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch2.chunkData.next_after)]]), {
         limit: 4
       })
     );
@@ -1425,6 +1439,159 @@ bucket_definitions:
     expect(test_utils.getBatchMeta(batch3)).toEqual(null);
   });
 
+  describe('batch has_more', () => {
+    const setup = async (options: BucketDataBatchOptions) => {
+      const sync_rules = test_utils.testRules(
+        `
+  bucket_definitions:
+    global1:
+      data:
+        - SELECT id, description FROM test WHERE bucket = 'global1'
+    global2:
+      data:
+        - SELECT id, description FROM test WHERE bucket = 'global2'
+  `
+      );
+      await using factory = await generateStorageFactory();
+      const bucketStorage = factory.getInstance(sync_rules);
+
+      const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        const sourceTable = TEST_TABLE;
+
+        for (let i = 1; i <= 10; i++) {
+          await batch.save({
+            sourceTable,
+            tag: storage.SaveOperationTag.INSERT,
+            after: {
+              id: `test${i}`,
+              description: `test${i}`,
+              bucket: i == 1 ? 'global1' : 'global2'
+            },
+            afterReplicaId: `test${i}`
+          });
+        }
+      });
+
+      const checkpoint = result!.flushed_op;
+      return await test_utils.fromAsync(
+        bucketStorage.getBucketDataBatch(
+          checkpoint,
+          new Map([
+            ['global1[]', 0n],
+            ['global2[]', 0n]
+          ]),
+          options
+        )
+      );
+    };
+
+    test('batch has_more (1)', async () => {
+      const batch = await setup({ limit: 5 });
+      expect(batch.length).toEqual(2);
+
+      expect(batch[0].chunkData.bucket).toEqual('global1[]');
+      expect(batch[1].chunkData.bucket).toEqual('global2[]');
+
+      expect(test_utils.getBatchData(batch[0])).toEqual([
+        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+      ]);
+
+      expect(test_utils.getBatchData(batch[1])).toEqual([
+        { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
+        { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
+        { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
+        { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }
+      ]);
+
+      expect(test_utils.getBatchMeta(batch[0])).toEqual({
+        after: '0',
+        has_more: false,
+        next_after: '1'
+      });
+
+      expect(test_utils.getBatchMeta(batch[1])).toEqual({
+        after: '0',
+        has_more: true,
+        next_after: '5'
+      });
+    });
+
+    test('batch has_more (2)', async () => {
+      const batch = await setup({ limit: 11 });
+      expect(batch.length).toEqual(2);
+
+      expect(batch[0].chunkData.bucket).toEqual('global1[]');
+      expect(batch[1].chunkData.bucket).toEqual('global2[]');
+
+      expect(test_utils.getBatchData(batch[0])).toEqual([
+        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+      ]);
+
+      expect(test_utils.getBatchData(batch[1])).toEqual([
+        { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
+        { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
+        { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
+        { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
+        { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
+        { op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
+        { op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
+        { op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
+        { op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
+      ]);
+
+      expect(test_utils.getBatchMeta(batch[0])).toEqual({
+        after: '0',
+        has_more: false,
+        next_after: '1'
+      });
+
+      expect(test_utils.getBatchMeta(batch[1])).toEqual({
+        after: '0',
+        has_more: false,
+        next_after: '10'
+      });
+    });
+
+    test('batch has_more (3)', async () => {
+      // 50 bytes is more than 1 row, less than 2 rows
+      const batch = await setup({ limit: 3, chunkLimitBytes: 50 });
+
+      expect(batch.length).toEqual(3);
+      expect(batch[0].chunkData.bucket).toEqual('global1[]');
+      expect(batch[1].chunkData.bucket).toEqual('global2[]');
+      expect(batch[2].chunkData.bucket).toEqual('global2[]');
+
+      expect(test_utils.getBatchData(batch[0])).toEqual([
+        { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+      ]);
+
+      expect(test_utils.getBatchData(batch[1])).toEqual([
+        { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 }
+      ]);
+      expect(test_utils.getBatchData(batch[2])).toEqual([
+        { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+      ]);
+
+      expect(test_utils.getBatchMeta(batch[0])).toEqual({
+        after: '0',
+        has_more: false,
+        next_after: '1'
+      });
+
+      expect(test_utils.getBatchMeta(batch[1])).toEqual({
+        after: '0',
+        has_more: true,
+        next_after: '2'
+      });
+
+      expect(test_utils.getBatchMeta(batch[2])).toEqual({
+        after: '2',
+        has_more: true,
+        next_after: '3'
+      });
+    });
+  });
+
   test('empty storage metrics', async () => {
     await using f = await generateStorageFactory({ dropAll: true });
     const metrics = await f.getStorageMetrics();
@@ -1711,4 +1878,54 @@ bucket_definitions:
       }
     });
   });
+
+  test('op_id initialization edge case', async () => {
+    // Test syncing a batch of data that is small in count,
+    // but large enough in size to be split over multiple returned chunks.
+    // Similar to the above test, but splits over 1MB chunks.
+    const sync_rules = test_utils.testRules(
+      `
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id FROM test
+        - SELECT id FROM test_ignore WHERE false
+  `
+    );
+    await using factory = await generateStorageFactory();
+    const bucketStorage = factory.getInstance(sync_rules);
+
+    const sourceTable = test_utils.makeTestTable('test', ['id']);
+    const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id']);
+
+    const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // This saves a record to current_data, but not bucket_data.
+      // This causes a checkpoint to be created without increasing the op_id sequence.
+      await batch.save({
+        sourceTable: sourceTableIgnore,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test1'
+        },
+        afterReplicaId: test_utils.rid('test1')
+      });
+    });
+
+    const checkpoint1 = result1!.flushed_op;
+
+    const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test2'
+        },
+        afterReplicaId: test_utils.rid('test2')
+      });
+    });
+
+    const checkpoint2 = result2!.flushed_op;
+    // we expect 0n and 1n, or 1n and 2n.
+    expect(checkpoint2).toBeGreaterThan(checkpoint1);
+  });
 }

package/src/tests/register-sync-tests.ts

@@ -160,8 +160,7 @@ bucket_definitions:
     expect(lines).toMatchSnapshot();
   });
 
-  // Temporarily skipped - interruption disabled
-  test.skip('sync interrupts low-priority buckets on new checkpoints', async () => {
+  test('sync interrupts low-priority buckets on new checkpoints', async () => {
     await using f = await factory();
 
     const syncRules = await f.updateSyncRules({
@@ -271,6 +270,141 @@ bucket_definitions:
     expect(sentRows).toBe(10002);
   });
 
+  test('sync interruptions with unrelated data', async () => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `
+bucket_definitions:
+  b0:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b1:
+    priority: 1
+    parameters: SELECT request.user_id() as user_id
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 5 AND description = bucket.user_id;
+    `
+    });
+
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // Initial data: Add one priority row and 10k low-priority rows.
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'highprio',
+          description: 'user_one'
+        },
+        afterReplicaId: 'highprio'
+      });
+      for (let i = 0; i < 10_000; i++) {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: `${i}`,
+            description: 'low prio'
+          },
+          afterReplicaId: `${i}`
+        });
+      }
+
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      syncContext,
+      bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: 'user_one' }, {}),
+      token: { sub: 'user_one', exp: Date.now() / 1000 + 100000 } as any
+    });
+
+    let sentCheckpoints = 0;
+    let completedCheckpoints = 0;
+    let sentRows = 0;
+
+    // Expected flow:
+    //  1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
+    //  2. We insert a new row that is not part of a bucket relevant to this stream.
+    //  3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
+    //  4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
+
+    for await (let next of stream) {
+      if (typeof next == 'string') {
+        next = JSON.parse(next);
+      }
+      if (typeof next === 'object' && next !== null) {
+        if ('partial_checkpoint_complete' in next) {
+          if (sentCheckpoints == 1) {
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+              // Add a high-priority row that doesn't affect this sync stream.
+              await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: storage.SaveOperationTag.INSERT,
+                after: {
+                  id: 'highprio2',
+                  description: 'user_two'
+                },
+                afterReplicaId: 'highprio2'
+              });
+
+              await batch.commit('0/2');
+            });
+          } else {
+            expect(sentCheckpoints).toBe(2);
+            expect(sentRows).toBe(10002);
+          }
+        }
+        if ('checkpoint' in next || 'checkpoint_diff' in next) {
+          sentCheckpoints += 1;
+        }
+
+        if ('data' in next) {
+          sentRows += next.data.data.length;
+        }
+        if ('checkpoint_complete' in next) {
+          completedCheckpoints++;
+          if (completedCheckpoints == 2) {
+            break;
+          }
+          if (completedCheckpoints == 1) {
+            expect(sentRows).toBe(10001);
+
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+              // Add a high-priority row that affects this sync stream.
+              await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: storage.SaveOperationTag.INSERT,
+                after: {
+                  id: 'highprio3',
+                  description: 'user_one'
+                },
+                afterReplicaId: 'highprio3'
+              });
+
+              await batch.commit('0/3');
+            });
+          }
+        }
+      }
+    }
+
+    expect(sentCheckpoints).toBe(2);
+    expect(sentRows).toBe(10002);
+  });
+
   test('sends checkpoint complete line for empty checkpoint', async () => {
     await using f = await factory();