@powersync/service-core-tests 0.0.0-dev-20250317122913 → 0.0.0-dev-20250325131118

package/src/tests/register-data-storage-tests.ts

@@ -1477,4 +1477,238 @@ bucket_definitions:
     expect(parsedSchema3).not.equals(parsedSchema2);
     expect(parsedSchema3.getSourceTables()[0].schema).equals('databasename');
   });
+
+  test('managed write checkpoints - checkpoint after write', async (context) => {
+    await using factory = await generateStorageFactory();
+    const r = await factory.configureSyncRules({
+      content: `
+bucket_definitions:
+  mybucket:
+    data: [] 
+    `,
+      validate: false
+    });
+    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    await bucketStorage.autoActivate();
+
+    const abortController = new AbortController();
+    context.onTestFinished(() => abortController.abort());
+    const iter = bucketStorage
+      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      [Symbol.asyncIterator]();
+
+    const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
+      heads: { '1': '5/0' },
+      user_id: 'user1'
+    });
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('5/0');
+    });
+
+    const result = await iter.next();
+    expect(result).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          checkpoint: 0n,
+          lsn: '5/0'
+        },
+        writeCheckpoint: writeCheckpoint
+      }
+    });
+  });
+
+  test('managed write checkpoints - write after checkpoint', async (context) => {
+    await using factory = await generateStorageFactory();
+    const r = await factory.configureSyncRules({
+      content: `
+bucket_definitions:
+  mybucket:
+    data: []
+    `,
+      validate: false
+    });
+    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    await bucketStorage.autoActivate();
+
+    const abortController = new AbortController();
+    context.onTestFinished(() => abortController.abort());
+    const iter = bucketStorage
+      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      [Symbol.asyncIterator]();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('5/0');
+    });
+
+    const result = await iter.next();
+    expect(result).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          checkpoint: 0n,
+          lsn: '5/0'
+        },
+        writeCheckpoint: null
+      }
+    });
+
+    const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
+      heads: { '1': '6/0' },
+      user_id: 'user1'
+    });
+    // We have to trigger a new keepalive after the checkpoint, at least to cover postgres storage.
+    // This is what is effetively triggered with RouteAPI.createReplicationHead().
+    // MongoDB storage doesn't explicitly need this anymore.
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('6/0');
+    });
+
+    let result2 = await iter.next();
+    if (result2.value?.base?.lsn == '5/0') {
+      // Events could arrive in a different order in some cases - this caters for it
+      result2 = await iter.next();
+    }
+    expect(result2).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          checkpoint: 0n,
+          lsn: '6/0'
+        },
+        writeCheckpoint: writeCheckpoint
+      }
+    });
+  });
+
+  test('custom write checkpoints - checkpoint after write', async (context) => {
+    await using factory = await generateStorageFactory();
+    const r = await factory.configureSyncRules({
+      content: `
+bucket_definitions:
+  mybucket:
+    data: []
+    `,
+      validate: false
+    });
+    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    await bucketStorage.autoActivate();
+    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+
+    const abortController = new AbortController();
+    context.onTestFinished(() => abortController.abort());
+    const iter = bucketStorage
+      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      [Symbol.asyncIterator]();
+
+    await bucketStorage.batchCreateCustomWriteCheckpoints([
+      {
+        checkpoint: 5n,
+        user_id: 'user1'
+      }
+    ]);
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('5/0');
+    });
+
+    const result = await iter.next();
+    expect(result).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          checkpoint: 0n,
+          lsn: '5/0'
+        },
+        writeCheckpoint: 5n
+      }
+    });
+  });
+
+  test('custom write checkpoints - write after checkpoint', async (context) => {
+    await using factory = await generateStorageFactory();
+    const r = await factory.configureSyncRules({
+      content: `
+bucket_definitions:
+  mybucket:
+    data: []
+    `,
+      validate: false
+    });
+    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    await bucketStorage.autoActivate();
+    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+
+    const abortController = new AbortController();
+    context.onTestFinished(() => abortController.abort());
+    const iter = bucketStorage
+      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      [Symbol.asyncIterator]();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('5/0');
+    });
+
+    const result = await iter.next();
+    expect(result).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          checkpoint: 0n,
+          lsn: '5/0'
+        },
+        writeCheckpoint: null
+      }
+    });
+
+    await bucketStorage.batchCreateCustomWriteCheckpoints([
+      {
+        checkpoint: 6n,
+        user_id: 'user1'
+      }
+    ]);
+    // We have to trigger a new keepalive after the checkpoint, at least to cover postgres storage.
+    // This is what is effetively triggered with RouteAPI.createReplicationHead().
+    // MongoDB storage doesn't explicitly need this anymore.
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('6/0');
+    });
+
+    let result2 = await iter.next();
+    expect(result2).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          checkpoint: 0n
+          // can be 5/0 or 6/0 - actual value not relevant for custom write checkpoints
+          // lsn: '6/0'
+        },
+        writeCheckpoint: 6n
+      }
+    });
+
+    await bucketStorage.batchCreateCustomWriteCheckpoints([
+      {
+        checkpoint: 7n,
+        user_id: 'user1'
+      }
+    ]);
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive('7/0');
+    });
+
+    let result3 = await iter.next();
+    expect(result3).toMatchObject({
+      done: false,
+      value: {
+        base: {
+          checkpoint: 0n
+          // can be 5/0, 6/0 or 7/0 - actual value not relevant for custom write checkpoints
+          // lsn: '7/0'
+        },
+        writeCheckpoint: 7n
+      }
+    });
+  });
 }

package/src/tests/register-sync-tests.ts

@@ -1,4 +1,11 @@
-import { storage, sync, utils } from '@powersync/service-core';
+import {
+  createCoreAPIMetrics,
+  storage,
+  StreamingSyncCheckpoint,
+  StreamingSyncCheckpointDiff,
+  sync,
+  utils
+} from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import path from 'path';
@@ -6,6 +13,7 @@ import * as timers from 'timers/promises';
 import { fileURLToPath } from 'url';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
+import { METRICS_HELPER } from '../test-utils/test-utils-index.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -30,7 +38,8 @@ export const SYNC_SNAPSHOT_PATH = path.resolve(__dirname, '../__snapshots/sync.t
  * ```
  */
 export function registerSyncTests(factory: storage.TestStorageFactory) {
-  const tracker = new sync.RequestTracker();
+  createCoreAPIMetrics(METRICS_HELPER.metricsEngine);
+  const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine);
   const syncContext = new sync.SyncContext({
     maxBuckets: 10,
     maxParameterQueryResults: 10,
@@ -398,7 +407,7 @@ bucket_definitions:
     expect(lines).toMatchSnapshot();
   });
 
-  test('sync updates to global data', async () => {
+  test('sync updates to global data', async (context) => {
     await using f = await factory();
 
     const syncRules = await f.updateSyncRules({
@@ -422,6 +431,9 @@ bucket_definitions:
       token: { exp: Date.now() / 1000 + 10 } as any
     });
     const iter = stream[Symbol.asyncIterator]();
+    context.onTestFinished(() => {
+      iter.return?.();
+    });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
@@ -456,11 +468,221 @@ bucket_definitions:
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
+  });
+
+  test('sync updates to parameter query only', async (context) => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `bucket_definitions:
+  by_user:
+    parameters: select users.id as user_id from users where users.id = request.user_id()
+    data:
+      - select * from lists where user_id = bucket.user_id
+`
+    });
+
+    const usersTable = test_utils.makeTestTable('users', ['id']);
+    const listsTable = test_utils.makeTestTable('lists', ['id']);
+
+    const bucketStorage = await f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    const stream = sync.streamResponse({
+      syncContext,
+      bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: 'user1' }, {}),
+      token: { exp: Date.now() / 1000 + 100 } as any
+    });
+    const iter = stream[Symbol.asyncIterator]();
+    context.onTestFinished(() => {
+      iter.return?.();
+    });
+
+    // Initial empty checkpoint
+    const checkpoint1 = await getCheckpointLines(iter);
+    expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([]);
+    expect(checkpoint1).toMatchSnapshot();
+
+    // Add user
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: usersTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'user1',
+          name: 'User 1'
+        },
+        afterReplicaId: 'user1'
+      });
+
+      await batch.commit('0/1');
+    });
+
+    const checkpoint2 = await getCheckpointLines(iter);
+    expect(
+      (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
+    ).toEqual(['by_user["user1"]']);
+    expect(checkpoint2).toMatchSnapshot();
+  });
+
+  test('sync updates to data query only', async (context) => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `bucket_definitions:
+  by_user:
+    parameters: select users.id as user_id from users where users.id = request.user_id()
+    data:
+      - select * from lists where user_id = bucket.user_id
+`
+    });
+
+    const usersTable = test_utils.makeTestTable('users', ['id']);
+    const listsTable = test_utils.makeTestTable('lists', ['id']);
+
+    const bucketStorage = await f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: usersTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'user1',
+          name: 'User 1'
+        },
+        afterReplicaId: 'user1'
+      });
+
+      await batch.commit('0/1');
+    });
+
+    const stream = sync.streamResponse({
+      syncContext,
+      bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: 'user1' }, {}),
+      token: { exp: Date.now() / 1000 + 100 } as any
+    });
+    const iter = stream[Symbol.asyncIterator]();
+    context.onTestFinished(() => {
+      iter.return?.();
+    });
+
+    const checkpoint1 = await getCheckpointLines(iter);
+    expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([
+      'by_user["user1"]'
+    ]);
+    expect(checkpoint1).toMatchSnapshot();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: listsTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'list1',
+          user_id: 'user1',
+          name: 'User 1'
+        },
+        afterReplicaId: 'list1'
+      });
+
+      await batch.commit('0/1');
+    });
+
+    const checkpoint2 = await getCheckpointLines(iter);
+    expect(
+      (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
+    ).toEqual(['by_user["user1"]']);
+    expect(checkpoint2).toMatchSnapshot();
+  });
+
+  test('sync updates to parameter query + data', async (context) => {
+    await using f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: `bucket_definitions:
+  by_user:
+    parameters: select users.id as user_id from users where users.id = request.user_id()
+    data:
+      - select * from lists where user_id = bucket.user_id
+`
+    });
+
+    const usersTable = test_utils.makeTestTable('users', ['id']);
+    const listsTable = test_utils.makeTestTable('lists', ['id']);
+
+    const bucketStorage = await f.getInstance(syncRules);
+    await bucketStorage.autoActivate();
+
+    const stream = sync.streamResponse({
+      syncContext,
+      bucketStorage,
+      syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: 'user1' }, {}),
+      token: { exp: Date.now() / 1000 + 100 } as any
+    });
+    const iter = stream[Symbol.asyncIterator]();
+    context.onTestFinished(() => {
+      iter.return?.();
+    });
+
+    // Initial empty checkpoint
+    expect(await getCheckpointLines(iter)).toMatchSnapshot();
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: listsTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'list1',
+          user_id: 'user1',
+          name: 'User 1'
+        },
+        afterReplicaId: 'list1'
+      });
 
-    iter.return?.();
+      await batch.save({
+        sourceTable: usersTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'user1',
+          name: 'User 1'
+        },
+        afterReplicaId: 'user1'
+      });
+
+      await batch.commit('0/1');
+    });
+
+    const checkpoint2 = await getCheckpointLines(iter);
+    expect(
+      (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
+    ).toEqual(['by_user["user1"]']);
+    expect(checkpoint2).toMatchSnapshot();
   });
 
-  test('expiring token', async () => {
+  test('expiring token', async (context) => {
     await using f = await factory();
 
     const syncRules = await f.updateSyncRules({
@@ -486,6 +708,9 @@ bucket_definitions:
       token: { exp: exp } as any
     });
     const iter = stream[Symbol.asyncIterator]();
+    context.onTestFinished(() => {
+      iter.return?.();
+    });
 
     const checkpoint = await getCheckpointLines(iter);
     expect(checkpoint).toMatchSnapshot();
@@ -494,7 +719,7 @@ bucket_definitions:
     expect(expLines).toMatchSnapshot();
   });
 
-  test('compacting data - invalidate checkpoint', async () => {
+  test('compacting data - invalidate checkpoint', async (context) => {
     // This tests a case of a compact operation invalidating a checkpoint in the
     // middle of syncing data.
     // This is expected to be rare in practice, but it is important to handle
@@ -548,6 +773,9 @@ bucket_definitions:
     });
 
     const iter = stream[Symbol.asyncIterator]();
+    context.onTestFinished(() => {
+      iter.return?.();
+    });
 
     // Only consume the first "checkpoint" message, and pause before receiving data.
     const lines = await consumeIterator(iter, { consume: false, isDone: (line) => (line as any)?.checkpoint != null });