@powersync/service-core-tests 0.14.0 → 0.15.0

package/src/tests/register-sync-tests.ts

@@ -14,14 +14,11 @@ import * as timers from 'timers/promises';
 import { fileURLToPath } from 'url';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
-import { METRICS_HELPER } from '../test-utils/test-utils-index.js';
-import { bucketRequest } from './util.js';
+import { bucketRequest, METRICS_HELPER } from '../test-utils/test-utils-index.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
-
 const BASIC_SYNC_RULES = `
 bucket_definitions:
   mybucket:
@@ -39,7 +36,15 @@ export const SYNC_SNAPSHOT_PATH = path.resolve(__dirname, '../__snapshots/sync.t
  * });
  * ```
  */
-export function registerSyncTests(factory: storage.TestStorageFactory, options: { storageVersion?: number } = {}) {
+export function registerSyncTests(
+  configOrFactory: storage.TestStorageConfig | storage.TestStorageFactory,
+  options: { storageVersion?: number; tableIdStrings?: boolean } = {}
+) {
+  const config: storage.TestStorageConfig =
+    typeof configOrFactory == 'function'
+      ? { factory: configOrFactory, tableIdStrings: options.tableIdStrings ?? true }
+      : configOrFactory;
+  const factory = config.factory;
   createCoreAPIMetrics(METRICS_HELPER.metricsEngine);
   const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine);
   const syncContext = new sync.SyncContext({
@@ -48,6 +53,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory, options:
     maxDataFetchConcurrency: 2
   });
 
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
   const updateSyncRules = (bucketStorageFactory: storage.BucketStorageFactory, updateOptions: { content: string }) => {
     return bucketStorageFactory.updateSyncRules(
       updateSyncRulesFromYaml(updateOptions.content, {
@@ -66,7 +72,9 @@ export function registerSyncTests(factory: storage.TestStorageFactory, options:
 
     const bucketStorage = f.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
+
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -127,7 +135,8 @@ bucket_definitions:
 
     const bucketStorage = f.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -189,6 +198,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // Initial data: Add one priority row and 10k low-priority rows.
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -299,6 +309,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // Initial data: Add one priority row and 10k low-priority rows.
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -440,6 +451,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // Initial data: Add one priority row and 10k low-priority rows.
       await batch.save({
         sourceTable: TEST_TABLE,
@@ -569,6 +581,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -633,6 +646,7 @@ bucket_definitions:
     const bucketStorage = await f.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -704,6 +718,7 @@ bucket_definitions:
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -772,12 +787,13 @@ bucket_definitions:
 `
     });
 
-    const usersTable = test_utils.makeTestTable('users', ['id']);
-    const listsTable = test_utils.makeTestTable('lists', ['id']);
+    const usersTable = test_utils.makeTestTable('users', ['id'], config);
+    const listsTable = test_utils.makeTestTable('lists', ['id'], config);
 
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -819,10 +835,11 @@ bucket_definitions:
       await batch.commit('0/1');
     });
 
+    const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual([bucketRequest(syncRules, 'by_user["user1"]')]);
+    ).toEqual([bucket]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
@@ -838,12 +855,13 @@ bucket_definitions:
 `
     });
 
-    const usersTable = test_utils.makeTestTable('users', ['id']);
-    const listsTable = test_utils.makeTestTable('lists', ['id']);
+    const usersTable = test_utils.makeTestTable('users', ['id'], config);
+    const listsTable = test_utils.makeTestTable('lists', ['id'], config);
 
     const bucketStorage = await f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: usersTable,
         tag: storage.SaveOperationTag.INSERT,
@@ -875,10 +893,9 @@ bucket_definitions:
       iter.return?.();
     });
 
+    const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
     const checkpoint1 = await getCheckpointLines(iter);
-    expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([
-      bucketRequest(syncRules, 'by_user["user1"]')
-    ]);
+    expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]);
     expect(checkpoint1).toMatchSnapshot();
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -899,7 +916,7 @@ bucket_definitions:
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual([bucketRequest(syncRules, 'by_user["user1"]')]);
+    ).toEqual([bucket]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
@@ -915,12 +932,13 @@ bucket_definitions:
 `
     });
 
-    const usersTable = test_utils.makeTestTable('users', ['id']);
-    const listsTable = test_utils.makeTestTable('lists', ['id']);
+    const usersTable = test_utils.makeTestTable('users', ['id'], config);
+    const listsTable = test_utils.makeTestTable('lists', ['id'], config);
 
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -946,6 +964,7 @@ bucket_definitions:
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: listsTable,
         tag: storage.SaveOperationTag.INSERT,
@@ -970,10 +989,11 @@ bucket_definitions:
       await batch.commit('0/1');
     });
 
+    const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual([bucketRequest(syncRules, 'by_user["user1"]')]);
+    ).toEqual([bucket]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
@@ -987,6 +1007,7 @@ bucket_definitions:
     const bucketStorage = await f.getInstance(syncRules);
     // Activate
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/0');
       await batch.keepalive('0/0');
     });
 
@@ -1032,6 +1053,7 @@ bucket_definitions:
     const bucketStorage = await f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -1087,6 +1109,7 @@ bucket_definitions:
     // This invalidates the checkpoint we've received above.
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.UPDATE,
@@ -1175,6 +1198,7 @@ bucket_definitions:
     const bucketStorage = f.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // <= the managed write checkpoint LSN below
       await batch.commit('0/1');
     });
@@ -1210,6 +1234,7 @@ bucket_definitions:
     });
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('0/1');
       // must be >= the managed write checkpoint LSN
       await batch.commit('1/0');
     });
@@ -1245,6 +1270,7 @@ config:
       const bucketStorage = f.getInstance(syncRules);
 
       await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        await batch.markAllSnapshotDone('0/1');
         await batch.save({
           sourceTable: TEST_TABLE,
           tag: storage.SaveOperationTag.INSERT,

package/src/tests/util.ts

@@ -1,24 +1,53 @@
 import { storage } from '@powersync/service-core';
-import { test_utils } from '../index.js';
+import {
+  ParameterIndexLookupCreator,
+  SourceTableInterface,
+  SqliteRow,
+  TablePattern
+} from '@powersync/service-sync-rules';
+import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
+import { bucketRequest } from '../test-utils/general-utils.js';
 
-export const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
-
-export function bucketRequest(syncRules: storage.PersistedSyncRulesContent, bucketName: string): string {
-  if (/^\d+#/.test(bucketName)) {
-    return bucketName;
-  }
-
-  const versionedBuckets = storage.STORAGE_VERSION_CONFIG[syncRules.storageVersion]?.versionedBuckets ?? false;
-  return versionedBuckets ? `${syncRules.id}#${bucketName}` : bucketName;
+export function bucketRequestMap(
+  syncRules: storage.PersistedSyncRulesContent,
+  buckets: Iterable<readonly [string, bigint]>
+): storage.BucketDataRequest[] {
+  return Array.from(buckets, ([bucketName, opId]) => bucketRequest(syncRules, bucketName, opId));
 }
 
-export function bucketRequests(syncRules: storage.PersistedSyncRulesContent, bucketNames: string[]): string[] {
-  return bucketNames.map((bucketName) => bucketRequest(syncRules, bucketName));
+export function bucketRequests(
+  syncRules: storage.PersistedSyncRulesContent,
+  bucketNames: string[]
+): storage.BucketChecksumRequest[] {
+  return bucketNames.map((bucketName) => {
+    const request = bucketRequest(syncRules, bucketName, 0n);
+    return { bucket: request.bucket, source: request.source };
+  });
 }
 
-export function bucketRequestMap(
-  syncRules: storage.PersistedSyncRulesContent,
-  buckets: Iterable<readonly [string, bigint]>
-): Map<string, bigint> {
-  return new Map(Array.from(buckets, ([bucketName, opId]) => [bucketRequest(syncRules, bucketName), opId]));
+const EMPTY_LOOKUP_SOURCE: ParameterIndexLookupCreator = {
+  get defaultLookupScope(): ParameterLookupScope {
+    return {
+      lookupName: 'lookup',
+      queryId: '0',
+      source: EMPTY_LOOKUP_SOURCE
+    };
+  },
+  getSourceTables(): Set<TablePattern> {
+    return new Set();
+  },
+  evaluateParameterRow(_sourceTable: SourceTableInterface, _row: SqliteRow) {
+    return [];
+  },
+  tableSyncsParameters(_table: SourceTableInterface): boolean {
+    return false;
+  }
+};
+
+export function parameterLookupScope(
+  lookupName: string,
+  queryId: string,
+  source: ParameterIndexLookupCreator = EMPTY_LOOKUP_SOURCE
+): ParameterLookupScope {
+  return { lookupName, queryId, source };
 }