@powersync/service-core-tests 0.0.0-dev-20250214100224 → 0.0.0-dev-20250227082606

package/src/tests/register-compacting-tests.ts

@@ -15,10 +15,7 @@ const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
  * compactTests(() => new MongoStorageFactory(), { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
  * ```
  */
-export function registerCompactTests<CompactOptions extends storage.CompactOptions = storage.CompactOptions>(
-  generateStorageFactory: storage.TestStorageFactory,
-  compactOptions: CompactOptions
-) {
+export function registerCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting (1)', async () => {
     const sync_rules = test_utils.testRules(`
 bucket_definitions:
@@ -87,7 +84,11 @@ bucket_definitions:
       }
     ]);
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]))
@@ -204,7 +205,11 @@ bucket_definitions:
       }
     ]);
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]))
@@ -285,7 +290,11 @@ bucket_definitions:
     });
     const checkpoint2 = result2!.flushed_op;
 
-    await bucketStorage.compact(compactOptions);
+    await bucketStorage.compact({
+      clearBatchLimit: 2,
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1
+    });
 
     const batchAfter = await test_utils.oneFromAsync(
       bucketStorage.getBucketDataBatch(checkpoint2, new Map([['global[]', '0']]))
@@ -307,4 +316,130 @@ bucket_definitions:
       checksum: 1874612650
     });
   });
+
+  test('compacting (4)', async () => {
+    const sync_rules = test_utils.testRules(/* yaml */
+    ` bucket_definitions:
+        grouped:
+          # The parameter query here is not important
+          # We specifically don't want to create bucket_parameter records here
+          # since the op_ids for bucket_data could vary between storage implementations.
+          parameters: select 'b' as b
+          data:
+            - select * from test where b = bucket.b`);
+
+    await using factory = await generateStorageFactory();
+    const bucketStorage = factory.getInstance(sync_rules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      /**
+       * Repeatedly create operations which fall into different buckets.
+       * The bucket operations are purposely interleaved as the op_id increases.
+       * A large amount of operations are created here.
+       * The configured window of compacting operations is 100. This means the initial window will
+       * contain operations from multiple buckets.
+       */
+      for (let count = 0; count < 100; count++) {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'start'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'intermediate'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't2',
+            b: 'b2',
+            value: 'start'
+          },
+          afterReplicaId: test_utils.rid('t2')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't1',
+            b: 'b1',
+            value: 'final'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.UPDATE,
+          after: {
+            id: 't2',
+            b: 'b2',
+            value: 'final'
+          },
+          afterReplicaId: test_utils.rid('t2')
+        });
+      }
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    await bucketStorage.compact({
+      clearBatchLimit: 100,
+      moveBatchLimit: 100,
+      moveBatchQueryLimit: 100 // Larger limit for a larger window of operations
+    });
+
+    const batchAfter = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(
+        checkpoint,
+        new Map([
+          ['grouped["b1"]', '0'],
+          ['grouped["b2"]', '0']
+        ])
+      )
+    );
+    const dataAfter = batchAfter.flatMap((b) => b.batch.data);
+
+    // The op_ids will vary between MongoDB and Postgres storage
+    expect(dataAfter).toMatchObject(
+      expect.arrayContaining([
+        { op_id: '497', op: 'CLEAR', checksum: -937074151 },
+        {
+          op_id: '499',
+          op: 'PUT',
+          object_type: 'test',
+          object_id: 't1',
+          checksum: 52221819,
+          subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+          data: '{"id":"t1","b":"b1","value":"final"}'
+        },
+        { op_id: '498', op: 'CLEAR', checksum: -234380197 },
+        {
+          op_id: '500',
+          op: 'PUT',
+          object_type: 'test',
+          object_id: 't2',
+          checksum: 2126669493,
+          subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+          data: '{"id":"t2","b":"b2","value":"final"}'
+        }
+      ])
+    );
+  });
 }

package/src/tests/register-data-storage-tests.ts

@@ -140,7 +140,7 @@ bucket_definitions:
     );
 
     await using factory = await generateStorageFactory();
-    await using bucketStorage = factory.getInstance(sync_rules);
+    const bucketStorage = factory.getInstance(sync_rules);
 
     const table = test_utils.makeTestTable('todos', ['id', 'list_id']);
 
@@ -394,13 +394,12 @@ bucket_definitions:
     const parameter_sets = await bucketStorage.getParameterSets(checkpoint, lookups);
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
 
-    const buckets = await sync_rules.queryBucketIds({
+    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
       getParameterSets(lookups) {
         return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
+      }
     });
-    expect(buckets).toEqual(['by_workspace["workspace1"]']);
+    expect(buckets).toEqual([{ bucket: 'by_workspace["workspace1"]', priority: 3 }]);
   });
 
   test('save and load parameters with dynamic global buckets', async () => {
@@ -466,14 +465,16 @@ bucket_definitions:
     parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
 
-    const buckets = await sync_rules.queryBucketIds({
+    const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
       getParameterSets(lookups) {
         return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
+      }
     });
-    buckets.sort();
-    expect(buckets).toEqual(['by_public_workspace["workspace1"]', 'by_public_workspace["workspace3"]']);
+    buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
+    expect(buckets).toEqual([
+      { bucket: 'by_public_workspace["workspace1"]', priority: 3 },
+      { bucket: 'by_public_workspace["workspace3"]', priority: 3 }
+    ]);
   });
 
   test('multiple parameter queries', async () => {
@@ -562,12 +563,13 @@ bucket_definitions:
     expect(parameter_sets2).toEqual([{ workspace_id: 'workspace3' }]);
 
     // Test final values - the important part
-    const buckets = await sync_rules.queryBucketIds({
-      getParameterSets(lookups) {
-        return bucketStorage.getParameterSets(checkpoint, lookups);
-      },
-      parameters
-    });
+    const buckets = (
+      await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return bucketStorage.getParameterSets(checkpoint, lookups);
+        }
+      })
+    ).map((e) => e.bucket);
     buckets.sort();
     expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']);
   });
@@ -1415,84 +1417,6 @@ bucket_definitions:
     expect(test_utils.getBatchMeta(batch3)).toEqual(null);
   });
 
-  test('batch should be disposed automatically', async () => {
-    const sync_rules = test_utils.testRules(`
-      bucket_definitions:
-        global:
-          data: [] 
-          `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
-
-    let isDisposed = false;
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      batch.registerListener({
-        disposed: () => {
-          isDisposed = true;
-        }
-      });
-    });
-    expect(isDisposed).true;
-
-    isDisposed = false;
-    let errorCaught = false;
-    try {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        batch.registerListener({
-          disposed: () => {
-            isDisposed = true;
-          }
-        });
-        throw new Error(`Testing exceptions`);
-      });
-    } catch (ex) {
-      errorCaught = true;
-      expect(ex.message.includes('Testing')).true;
-    }
-    expect(errorCaught).true;
-    expect(isDisposed).true;
-  });
-
-  test('batch should be disposed automatically', async () => {
-    const sync_rules = test_utils.testRules(`
-      bucket_definitions:
-        global:
-          data: [] 
-          `);
-
-    await using factory = await generateStorageFactory();
-    const bucketStorage = factory.getInstance(sync_rules);
-
-    let isDisposed = false;
-    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      batch.registerListener({
-        disposed: () => {
-          isDisposed = true;
-        }
-      });
-    });
-    expect(isDisposed).true;
-
-    isDisposed = false;
-    let errorCaught = false;
-    try {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        batch.registerListener({
-          disposed: () => {
-            isDisposed = true;
-          }
-        });
-        throw new Error(`Testing exceptions`);
-      });
-    } catch (ex) {
-      errorCaught = true;
-      expect(ex.message.includes('Testing')).true;
-    }
-    expect(errorCaught).true;
-    expect(isDisposed).true;
-  });
-
   test('empty storage metrics', async () => {
     await using f = await generateStorageFactory({ dropAll: true });
     const metrics = await f.getStorageMetrics();