@powersync/service-core-tests 0.13.2 → 0.15.0

package/src/tests/register-sync-tests.ts

@@ -5,22 +5,20 @@ import {
   StreamingSyncCheckpoint,
   StreamingSyncCheckpointDiff,
   sync,
+  updateSyncRulesFromYaml,
   utils
 } from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
-import { BucketSourceType, RequestParameters } from '@powersync/service-sync-rules';
 import path from 'path';
 import * as timers from 'timers/promises';
 import { fileURLToPath } from 'url';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
-import { METRICS_HELPER } from '../test-utils/test-utils-index.js';
+import { bucketRequest, METRICS_HELPER } from '../test-utils/test-utils-index.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
-
 const BASIC_SYNC_RULES = `
 bucket_definitions:
   mybucket:
@@ -38,7 +36,15 @@ export const SYNC_SNAPSHOT_PATH = path.resolve(__dirname, '../__snapshots/sync.t
  * });
  * ```
  */
-export function registerSyncTests(factory: storage.TestStorageFactory) {
+export function registerSyncTests(
+  configOrFactory: storage.TestStorageConfig | storage.TestStorageFactory,
+  options: { storageVersion?: number; tableIdStrings?: boolean } = {}
+) {
+  const config: storage.TestStorageConfig =
+    typeof configOrFactory == 'function'
+      ? { factory: configOrFactory, tableIdStrings: options.tableIdStrings ?? true }
+      : configOrFactory;
+  const factory = config.factory;
   createCoreAPIMetrics(METRICS_HELPER.metricsEngine);
   const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine);
   const syncContext = new sync.SyncContext({
@@ -47,16 +53,28 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     maxDataFetchConcurrency: 2
   });
 
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
+  const updateSyncRules = (bucketStorageFactory: storage.BucketStorageFactory, updateOptions: { content: string }) => {
+    return bucketStorageFactory.updateSyncRules(
+      updateSyncRulesFromYaml(updateOptions.content, {
+        validate: true,
+        storageVersion: options.storageVersion
+      })
+    );
+  };
+
   test('sync global data', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
     const bucketStorage = f.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
+
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -101,7 +119,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
   test('sync buckets in order', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0:
@@ -117,7 +135,8 @@ bucket_definitions:
 
     const bucketStorage = f.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -162,7 +181,7 @@ bucket_definitions:
   test('sync interrupts low-priority buckets on new checkpoints', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0:
@@ -179,6 +198,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // Initial data: Add one priority row and 10k low-priority rows.
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -271,7 +291,7 @@ bucket_definitions:
   test('sync interruptions with unrelated data', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0:
@@ -289,6 +309,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // Initial data: Add one priority row and 10k low-priority rows.
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -409,7 +430,7 @@ bucket_definitions:
     // then interrupt checkpoint with new data for all buckets
     // -> data for all buckets should be sent in the new checkpoint
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0a:
@@ -430,6 +451,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // Initial data: Add one priority row and 10k low-priority rows.
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -553,12 +575,13 @@ bucket_definitions:
   test('sends checkpoint complete line for empty checkpoint', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -616,13 +639,14 @@ bucket_definitions:
   test('sync legacy non-raw data', async () => {
     const f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
     const bucketStorage = await f.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -660,7 +684,7 @@ bucket_definitions:
   test('expired token', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -687,13 +711,14 @@ bucket_definitions:
   test('sync updates to global data', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -753,7 +778,7 @@ bucket_definitions:
   test('sync updates to parameter query only', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `bucket_definitions:
   by_user:
     parameters: select users.id as user_id from users where users.id = request.user_id()
@@ -762,12 +787,13 @@ bucket_definitions:
 `
     });
 
-    const usersTable = test_utils.makeTestTable('users', ['id']);
-    const listsTable = test_utils.makeTestTable('lists', ['id']);
+    const usersTable = test_utils.makeTestTable('users', ['id'], config);
+    const listsTable = test_utils.makeTestTable('lists', ['id'], config);
 
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -809,17 +835,18 @@ bucket_definitions:
       await batch.commit('0/1');
     });
 
+    const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual(['by_user["user1"]']);
+    ).toEqual([bucket]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
   test('sync updates to data query only', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `bucket_definitions:
   by_user:
     parameters: select users.id as user_id from users where users.id = request.user_id()
@@ -828,12 +855,13 @@ bucket_definitions:
 `
     });
 
-    const usersTable = test_utils.makeTestTable('users', ['id']);
-    const listsTable = test_utils.makeTestTable('lists', ['id']);
+    const usersTable = test_utils.makeTestTable('users', ['id'], config);
+    const listsTable = test_utils.makeTestTable('lists', ['id'], config);
 
     const bucketStorage = await f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: usersTable,
         tag: storage.SaveOperationTag.INSERT,
@@ -865,10 +893,9 @@ bucket_definitions:
       iter.return?.();
     });
 
+    const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
     const checkpoint1 = await getCheckpointLines(iter);
-    expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([
-      'by_user["user1"]'
-    ]);
+    expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]);
     expect(checkpoint1).toMatchSnapshot();
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -889,14 +916,14 @@ bucket_definitions:
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual(['by_user["user1"]']);
+    ).toEqual([bucket]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
   test('sync updates to parameter query + data', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `bucket_definitions:
   by_user:
     parameters: select users.id as user_id from users where users.id = request.user_id()
@@ -905,12 +932,13 @@ bucket_definitions:
 `
     });
 
-    const usersTable = test_utils.makeTestTable('users', ['id']);
-    const listsTable = test_utils.makeTestTable('lists', ['id']);
+    const usersTable = test_utils.makeTestTable('users', ['id'], config);
+    const listsTable = test_utils.makeTestTable('lists', ['id'], config);
 
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -936,6 +964,7 @@ bucket_definitions:
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: listsTable,
         tag: storage.SaveOperationTag.INSERT,
@@ -960,23 +989,25 @@ bucket_definitions:
       await batch.commit('0/1');
     });
 
+    const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual(['by_user["user1"]']);
+    ).toEqual([bucket]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
   test('expiring token', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -1015,13 +1046,14 @@ bucket_definitions:
 
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
     const bucketStorage = await f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -1077,6 +1109,7 @@ bucket_definitions:
     // This invalidates the checkpoint we've received above.
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.UPDATE,
@@ -1158,13 +1191,14 @@ bucket_definitions:
   test('write checkpoint', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // <= the managed write checkpoint LSN below
       await batch.commit('0/1');
     });
@@ -1200,6 +1234,7 @@ bucket_definitions:
     });
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // must be >= the managed write checkpoint LSN
       await batch.commit('1/0');
     });
@@ -1229,12 +1264,13 @@ config:
 `;
 
     for (let i = 0; i < 2; i++) {
-      const syncRules = await f.updateSyncRules({
+      const syncRules = await updateSyncRules(f, {
         content: rules
       });
       const bucketStorage = f.getInstance(syncRules);
 
       await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        await batch.markAllSnapshotDone('0/1');
         await batch.save({
           sourceTable: TEST_TABLE,
           tag: storage.SaveOperationTag.INSERT,

package/src/tests/util.ts

@@ -1,3 +1,53 @@
-import { test_utils } from '../index.js';
+import { storage } from '@powersync/service-core';
+import {
+  ParameterIndexLookupCreator,
+  SourceTableInterface,
+  SqliteRow,
+  TablePattern
+} from '@powersync/service-sync-rules';
+import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
+import { bucketRequest } from '../test-utils/general-utils.js';
 
-export const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
+export function bucketRequestMap(
+  syncRules: storage.PersistedSyncRulesContent,
+  buckets: Iterable<readonly [string, bigint]>
+): storage.BucketDataRequest[] {
+  return Array.from(buckets, ([bucketName, opId]) => bucketRequest(syncRules, bucketName, opId));
+}
+
+export function bucketRequests(
+  syncRules: storage.PersistedSyncRulesContent,
+  bucketNames: string[]
+): storage.BucketChecksumRequest[] {
+  return bucketNames.map((bucketName) => {
+    const request = bucketRequest(syncRules, bucketName, 0n);
+    return { bucket: request.bucket, source: request.source };
+  });
+}
+
+const EMPTY_LOOKUP_SOURCE: ParameterIndexLookupCreator = {
+  get defaultLookupScope(): ParameterLookupScope {
+    return {
+      lookupName: 'lookup',
+      queryId: '0',
+      source: EMPTY_LOOKUP_SOURCE
+    };
+  },
+  getSourceTables(): Set<TablePattern> {
+    return new Set();
+  },
+  evaluateParameterRow(_sourceTable: SourceTableInterface, _row: SqliteRow) {
+    return [];
+  },
+  tableSyncsParameters(_table: SourceTableInterface): boolean {
+    return false;
+  }
+};
+
+export function parameterLookupScope(
+  lookupName: string,
+  queryId: string,
+  source: ParameterIndexLookupCreator = EMPTY_LOOKUP_SOURCE
+): ParameterLookupScope {
+  return { lookupName, queryId, source };
+}