@powersync/service-core-tests 0.5.0 → 0.6.1

package/src/tests/register-compacting-tests.ts

@@ -15,10 +15,7 @@ const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
  * compactTests(() => new MongoStorageFactory(), { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
  * ```
  */
-export function registerCompactTests<CompactOptions extends storage.CompactOptions = storage.CompactOptions>(
-  generateStorageFactory: storage.TestStorageFactory,
-  compactOptions: CompactOptions
-) {
+export function registerCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting (1)', async () => {
     const sync_rules = test_utils.testRules(`
 bucket_definitions:
@@ -87,7 +84,11 @@ bucket_definitions:
       }
     ]);
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]))
@@ -204,7 +205,11 @@ bucket_definitions:
       }
     ]);
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]))
@@ -285,7 +290,11 @@ bucket_definitions:
     });
     const checkpoint2 = result2!.flushed_op;
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', '0']]))
@@ -307,4 +316,130 @@ bucket_definitions:
       checksum: 1874612650
     });
   });
+
+  test('compacting (4)', async () => {
+    const sync_rules = test_utils.testRules(/* yaml */
+    ` bucket_definitions:
+        grouped:
+          # The parameter query here is not important
+          # We specifically don't want to create bucket_parameter records here
+          # since the op_ids for bucket_data could vary between storage implementations.
+          parameters: select 'b' as b
+          data:
+            - select * from test where b = bucket.b`);
+
+    await using factory = await generateStorageFactory();
+    const bucketStorage = factory.getInstance(sync_rules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      /**
+       * Repeatedly create operations which fall into different buckets.
+       * The bucket operations are purposely interleaved as the op_id increases.
+       * A large amount of operations are created here.
+       * The configured window of compacting operations is 100. This means the initial window will
+       * contain operations from multiple buckets.
+       */
+      for (let count = 0; count < 100; count++) {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'start'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'intermediate'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't2',
+            b: 'b2',
+            value: 'start'
+          },
+          afterReplicaId: test_utils.rid('t2')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'final'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't2',
+            b: 'b2',
+            value: 'final'
+          },
+          afterReplicaId: test_utils.rid('t2')
+        });
+      }
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    await bucketStorage.compact({
+      clearBatchLimit: 100,
+      moveBatchLimit: 100,
+      moveBatchQueryLimit: 100 // Larger limit for a larger window of operations
+    });
+
+    const batchAfter = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(
+        checkpoint,
+        new Map([
+          ['grouped["b1"]', '0'],
+          ['grouped["b2"]', '0']
+        ])
+      )
+    );
+    const dataAfter = batchAfter.flatMap((b) => b.batch.data);
+
+    // The op_ids will vary between MongoDB and Postgres storage
+    expect(dataAfter).toMatchObject(
+      expect.arrayContaining([
+        { op_id: '497', op: 'CLEAR', checksum: -937074151 },
+        {
+          op_id: '499',
+          op: 'PUT',
+          object_type: 'test',
+          object_id: 't1',
+          checksum: 52221819,
+          subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+          data: '{"id":"t1","b":"b1","value":"final"}'
+        },
+        { op_id: '498', op: 'CLEAR', checksum: -234380197 },
+        {
+          op_id: '500',
+          op: 'PUT',
+          object_type: 'test',
+          object_id: 't2',
+          checksum: 2126669493,
+          subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+          data: '{"id":"t2","b":"b2","value":"final"}'
+        }
+      ])
+    );
+  });
 }

package/src/tests/register-sync-tests.ts

@@ -253,6 +253,69 @@ bucket_definitions:
     expect(sentRows).toBe(10002);
   });
 
+  test('sends checkpoint complete line for empty checkpoint', async () => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: BASIC_SYNC_RULES
+    });
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1',
+          description: 'sync'
+        },
+        afterReplicaId: 't1'
+      });
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 100000 } as any
+    });
+
+    const lines: any[] = [];
+    let receivedCompletions = 0;
+
+    for await (let next of stream) {
+      if (typeof next == 'string') {
+        next = JSON.parse(next);
+      }
+      lines.push(next);
+
+      if (typeof next === 'object' && next !== null) {
+        if ('checkpoint_complete' in next) {
+          receivedCompletions++;
+          if (receivedCompletions == 1) {
+            // Trigger an empty bucket update.
+            await bucketStorage.createManagedWriteCheckpoint({user_id: '', heads: {'1': '1/0'}});
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+              await batch.commit('1/0');
+            });
+          } else {
+            break;
+          }
+        }
+      }
+    }
+
+    expect(lines).toMatchSnapshot();
+  });
+
   test('sync legacy non-raw data', async () => {
     const f = await factory();