@powersync/service-core-tests 0.4.1 → 0.6.0

package/src/tests/register-compacting-tests.ts

@@ -15,10 +15,7 @@ const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
  * compactTests(() => new MongoStorageFactory(), { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
  * ```
  */
-export function registerCompactTests<CompactOptions extends storage.CompactOptions = storage.CompactOptions>(
-  generateStorageFactory: storage.TestStorageFactory,
-  compactOptions: CompactOptions
-) {
+export function registerCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting (1)', async () => {
     const sync_rules = test_utils.testRules(`
 bucket_definitions:
@@ -87,7 +84,11 @@ bucket_definitions:
       }
     ]);
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]))
@@ -204,7 +205,11 @@ bucket_definitions:
       }
     ]);
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]))
@@ -285,7 +290,11 @@ bucket_definitions:
     });
     const checkpoint2 = result2!.flushed_op;
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', '0']]))
@@ -307,4 +316,130 @@ bucket_definitions:
       checksum: 1874612650
     });
   });
+
+  test('compacting (4)', async () => {
+    const sync_rules = test_utils.testRules(/* yaml */
+    ` bucket_definitions:
+        grouped:
+          # The parameter query here is not important
+          # We specifically don't want to create bucket_parameter records here
+          # since the op_ids for bucket_data could vary between storage implementations.
+          parameters: select 'b' as b
+          data:
+            - select * from test where b = bucket.b`);
+
+    await using factory = await generateStorageFactory();
+    const bucketStorage = factory.getInstance(sync_rules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      /**
+       * Repeatedly create operations which fall into different buckets.
+       * The bucket operations are purposely interleaved as the op_id increases.
+       * A large amount of operations are created here.
+       * The configured window of compacting operations is 100. This means the initial window will
+       * contain operations from multiple buckets.
+       */
+      for (let count = 0; count < 100; count++) {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'start'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'intermediate'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't2',
+            b: 'b2',
+            value: 'start'
+          },
+          afterReplicaId: test_utils.rid('t2')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'final'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't2',
+            b: 'b2',
+            value: 'final'
+          },
+          afterReplicaId: test_utils.rid('t2')
+        });
+      }
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    await bucketStorage.compact({
+      clearBatchLimit: 100,
+      moveBatchLimit: 100,
+      moveBatchQueryLimit: 100 // Larger limit for a larger window of operations
+    });
+
+    const batchAfter = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(
+        checkpoint,
+        new Map([
+          ['grouped["b1"]', '0'],
+          ['grouped["b2"]', '0']
+        ])
+      )
+    );
+    const dataAfter = batchAfter.flatMap((b) => b.batch.data);
+
+    // The op_ids will vary between MongoDB and Postgres storage
+    expect(dataAfter).toMatchObject(
+      expect.arrayContaining([
+        { op_id: '497', op: 'CLEAR', checksum: -937074151 },
+        {
+          op_id: '499',
+          op: 'PUT',
+          object_type: 'test',
+          object_id: 't1',
+          checksum: 52221819,
+          subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+          data: '{"id":"t1","b":"b1","value":"final"}'
+        },
+        { op_id: '498', op: 'CLEAR', checksum: -234380197 },
+        {
+          op_id: '500',
+          op: 'PUT',
+          object_type: 'test',
+          object_id: 't2',
+          checksum: 2126669493,
+          subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+          data: '{"id":"t2","b":"b2","value":"final"}'
+        }
+      ])
+    );
+  });
 }

package/src/tests/register-data-storage-tests.ts

@@ -140,7 +140,7 @@ bucket_definitions:
     );
 
     await using factory = await generateStorageFactory();
-    await using bucketStorage = factory.getInstance(sync_rules);
+    const bucketStorage = factory.getInstance(sync_rules);
 
     const table = test_utils.makeTestTable('todos', ['id', 'list_id']);
 
@@ -394,13 +394,12 @@ bucket_definitions:
     const parameter_sets = await bucketStorage.getParameterSets(checkpoint, lookups);
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const buckets = await sync_rules.queryBucketIds({
+    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
       getParameterSets(lookups) {
         return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
+      }
     });
-    expect(buckets).toEqual(['by_workspace["workspace1"]']);
+    expect(buckets).toEqual([{ bucket: 'by_workspace["workspace1"]', priority: 3 }]);
   });
 
   test('save and load parameters with dynamic global buckets', async () => {
@@ -466,14 +465,16 @@ bucket_definitions:
     parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
 
-    const buckets = await sync_rules.queryBucketIds({
+    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
       getParameterSets(lookups) {
         return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
+      }
     });
-    buckets.sort();
-    expect(buckets).toEqual(['by_public_workspace["workspace1"]', 'by_public_workspace["workspace3"]']);
+    buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
+    expect(buckets).toEqual([
+      { bucket: 'by_public_workspace["workspace1"]', priority: 3 },
+      { bucket: 'by_public_workspace["workspace3"]', priority: 3 }
+    ]);
   });
 
   test('multiple parameter queries', async () => {
@@ -562,12 +563,13 @@ bucket_definitions:
     expect(parameter_sets2).toEqual([{ workspace_id: 'workspace3' }]);
 
     // Test final values - the important part
-    const buckets = await sync_rules.queryBucketIds({
-      getParameterSets(lookups) {
-        return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
-    });
+    const buckets = (
+      await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return bucketStorage.getParameterSets(checkpoint, lookups);
+        }
+      })
+    ).map((e) => e.bucket);
     buckets.sort();
     expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']);
   });
@@ -1415,84 +1417,6 @@ bucket_definitions:
     expect(test_utils.getBatchMeta(batch3)).toEqual(null);
   });
 
-  test('batch should be disposed automatically', async () => {
-    const sync_rules = test_utils.testRules(`
-      bucket_definitions:
-        global:
-          data: [] 
-          `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
-
-    let isDisposed = false;
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      batch.registerListener({
-        disposed: () => {
-          isDisposed = true;
-        }
-      });
-    });
-    expect(isDisposed).true;
-
-    isDisposed = false;
-    let errorCaught = false;
-    try {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        batch.registerListener({
-          disposed: () => {
-            isDisposed = true;
-          }
-        });
-        throw new Error(`Testing exceptions`);
-      });
-    } catch (ex) {
-      errorCaught = true;
-      expect(ex.message.includes('Testing')).true;
-    }
-    expect(errorCaught).true;
-    expect(isDisposed).true;
-  });
-
-  test('batch should be disposed automatically', async () => {
-    const sync_rules = test_utils.testRules(`
-      bucket_definitions:
-        global:
-          data: [] 
-          `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
-
-    let isDisposed = false;
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      batch.registerListener({
-        disposed: () => {
-          isDisposed = true;
-        }
-      });
-    });
-    expect(isDisposed).true;
-
-    isDisposed = false;
-    let errorCaught = false;
-    try {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        batch.registerListener({
-          disposed: () => {
-            isDisposed = true;
-          }
-        });
-        throw new Error(`Testing exceptions`);
-      });
-    } catch (ex) {
-      errorCaught = true;
-      expect(ex.message.includes('Testing')).true;
-    }
-    expect(errorCaught).true;
-    expect(isDisposed).true;
-  });
-
   test('empty storage metrics', async () => {
     await using f = await generateStorageFactory({ dropAll: true });
     const metrics = await f.getStorageMetrics();

package/src/tests/register-sync-tests.ts

@@ -67,13 +67,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -83,6 +83,239 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     expect(lines).toMatchSnapshot();
   });
 
+  test('sync buckets in order', async () => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `
+bucket_definitions:
+  b0:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 2;
+  b1:
+    priority: 1
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 2;
+    `
+    });
+
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1',
+          description: 'Test 1'
+        },
+        afterReplicaId: 't1'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'earlier',
+          description: 'Test 2'
+        },
+        afterReplicaId: 'earlier'
+      });
+
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    const lines = await consumeCheckpointLines(stream);
+    expect(lines).toMatchSnapshot();
+  });
+
+  test('sync interrupts low-priority buckets on new checkpoints', async () => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `
+bucket_definitions:
+  b0:
+    priority: 2
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) <= 5;
+  b1:
+    priority: 1
+    data:
+      - SELECT * FROM test WHERE LENGTH(id) > 5;
+    `
+    });
+
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // Initial data: Add one priority row and 10k low-priority rows.
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'highprio',
+          description: 'High priority row'
+        },
+        afterReplicaId: 'highprio'
+      });
+      for (let i = 0; i < 10_000; i++) {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: `${i}`,
+            description: 'low prio'
+          },
+          afterReplicaId: `${i}`
+        });
+      }
+
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    let sentCheckpoints = 0;
+    let sentRows = 0;
+
+    for await (let next of stream) {
+      if (typeof next == 'string') {
+        next = JSON.parse(next);
+      }
+      if (typeof next === 'object' && next !== null) {
+        if ('partial_checkpoint_complete' in next) {
+          if (sentCheckpoints == 1) {
+            // Save new data to interrupt the low-priority sync.
+
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+              // Add another high-priority row. This should interrupt the long-running low-priority sync.
+              await batch.save({
+                sourceTable: TEST_TABLE,
+                tag: storage.SaveOperationTag.INSERT,
+                after: {
+                  id: 'highprio2',
+                  description: 'Another high-priority row'
+                },
+                afterReplicaId: 'highprio2'
+              });
+
+              await batch.commit('0/2');
+            });
+          } else {
+            // Low-priority sync from the first checkpoint was interrupted. This should not happen before
+            // 1000 low-priority items were synchronized.
+            expect(sentCheckpoints).toBe(2);
+            expect(sentRows).toBeGreaterThan(1000);
+          }
+        }
+        if ('checkpoint' in next || 'checkpoint_diff' in next) {
+          sentCheckpoints += 1;
+        }
+
+        if ('data' in next) {
+          sentRows += next.data.data.length;
+        }
+        if ('checkpoint_complete' in next) {
+          break;
+        }
+      }
+    }
+
+    expect(sentCheckpoints).toBe(2);
+    expect(sentRows).toBe(10002);
+  });
+
+  test('sends checkpoint complete line for empty checkpoint', async () => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: BASIC_SYNC_RULES
+    });
+    const bucketStorage = f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1',
+          description: 'sync'
+        },
+        afterReplicaId: 't1'
+      });
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 100000 } as any
+    });
+
+    const lines: any[] = [];
+    let receivedCompletions = 0;
+
+    for await (let next of stream) {
+      if (typeof next == 'string') {
+        next = JSON.parse(next);
+      }
+      lines.push(next);
+
+      if (typeof next === 'object' && next !== null) {
+        if ('checkpoint_complete' in next) {
+          receivedCompletions++;
+          if (receivedCompletions == 1) {
+            // Trigger an empty bucket update.
+            await bucketStorage.createManagedWriteCheckpoint({user_id: '', heads: {'1': '1/0'}});
+            await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+              await batch.commit('1/0');
+            });
+          } else {
+            break;
+          }
+        }
+      }
+    }
+
+    expect(lines).toMatchSnapshot();
+  });
+
   test('sync legacy non-raw data', async () => {
     const f = await factory();
 
@@ -109,13 +342,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: false
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -134,17 +367,17 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules);
-    await storage.autoActivate();
+    const bucketStorage = await f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
@@ -165,13 +398,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     await bucketStorage.autoActivate();
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -222,19 +455,19 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules);
-    await storage.autoActivate();
+    const bucketStorage = await f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
 
     const exp = Date.now() / 1000 + 0.1;
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
@@ -288,13 +521,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const stream = sync.streamResponse({
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -411,13 +644,13 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     });
 
     const params: sync.SyncStreamParameters = {
-      storage: f,
+      bucketStorage: bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
       params: {
         buckets: [],
         include_checksum: true,
         raw_data: true
       },
-      parseOptions: test_utils.PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: 'test' }, {}),
       token: { sub: 'test', exp: Date.now() / 1000 + 10 } as any