@powersync/service-core-tests 0.14.0 → 0.15.0

package/src/test-utils/general-utils.ts

@@ -1,4 +1,4 @@
-import { storage, utils } from '@powersync/service-core';
+import { BucketDataRequest, InternalOpId, storage, utils } from '@powersync/service-core';
 import { GetQuerierOptions, RequestParameters } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 
@@ -14,9 +14,14 @@ export const BATCH_OPTIONS: storage.StartBatchOptions = {
   storeCurrentData: true
 };
 
-export function makeTestTable(name: string, replicaIdColumns?: string[] | undefined) {
+export function makeTestTable(
+  name: string,
+  replicaIdColumns?: string[] | undefined,
+  options?: { tableIdStrings: boolean }
+) {
   const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(','));
-  const id = new bson.ObjectId('6544e3899293153fa7b38331');
+  const id =
+    options?.tableIdStrings == false ? new bson.ObjectId('6544e3899293153fa7b38331') : '6544e3899293153fa7b38331';
   return new storage.SourceTable({
     id: id,
     connectionTag: storage.SourceTable.DEFAULT_TAG,
@@ -45,6 +50,39 @@ export function getBatchData(
   });
 }
 
+function isParsedSyncRules(
+  syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules
+): syncRules is storage.PersistedSyncRules {
+  return (syncRules as storage.PersistedSyncRules).sync_rules !== undefined;
+}
+
+/**
+ * Bucket names no longer purely depend on the sync rules.
+ * This converts a bucket name like "global[]" into the actual bucket name, for use in tests.
+ */
+export function bucketRequest(
+  syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules,
+  bucket: string,
+  start?: InternalOpId | string | number
+): BucketDataRequest {
+  const parsed = isParsedSyncRules(syncRules) ? syncRules : syncRules.parsed(PARSE_OPTIONS);
+  const hydrationState = parsed.hydrationState;
+  const parameterStart = bucket.indexOf('[');
+  const definitionName = bucket.substring(0, parameterStart);
+  const parameters = bucket.substring(parameterStart);
+  const source = parsed.sync_rules.config.bucketDataSources.find((b) => b.uniqueName === definitionName);
+
+  if (source == null) {
+    throw new Error(`Failed to find global bucket ${bucket}`);
+  }
+  const bucketName = hydrationState.getBucketSourceScope(source).bucketPrefix + parameters;
+  return {
+    bucket: bucketName,
+    start: BigInt(start ?? 0n),
+    source: source
+  };
+}
+
 export function getBatchMeta(
   batch: utils.SyncBucketData[] | storage.SyncBucketDataChunk[] | storage.SyncBucketDataChunk
 ) {

package/src/tests/register-compacting-tests.ts

@@ -1,11 +1,13 @@
-import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
-import { bucketRequest, bucketRequestMap, bucketRequests } from './util.js';
+import { bucketRequest } from '../test-utils/test-utils-index.js';
+import { bucketRequestMap, bucketRequests } from './util.js';
 
-const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
+export function registerCompactTests(config: storage.TestStorageConfig) {
+  const generateStorageFactory = config.factory;
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
 
-export function registerCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting (1)', async () => {
     await using factory = await generateStorageFactory();
     const syncRules = await factory.updateSyncRules(
@@ -18,6 +20,7 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -50,27 +53,24 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const batchBefore = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
-    );
+    const request = bucketRequest(syncRules, 'global[]');
+
+    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
 
     expect(dataBefore).toMatchObject([
       {
-        checksum: 2634521662,
         object_id: 't1',
         op: 'PUT',
         op_id: '1'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
         op: 'PUT',
         op_id: '2'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
         op: 'PUT',
         op_id: '3'
@@ -86,41 +86,30 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
-    );
+    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataAfter = batchAfter.chunkData.data;
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
     bucketStorage.clearChecksumCache();
-    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
+    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, [request]);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
+      dataBefore[0],
       {
-        checksum: 2634521662,
-        object_id: 't1',
-        op: 'PUT',
-        op_id: '1'
-      },
-      {
-        checksum: 4243212114,
+        checksum: dataBefore[1].checksum,
         op: 'MOVE',
         op_id: '2'
       },
       {
-        checksum: 4243212114,
+        checksum: dataBefore[2].checksum,
         object_id: 't2',
         op: 'PUT',
         op_id: '3'
       }
     ]);
 
-    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual(
-      checksumBefore.get(bucketRequest(syncRules, 'global[]'))
-    );
-    expect(checksumAfter2.get(bucketRequest(syncRules, 'global[]'))).toEqual(
-      checksumBefore.get(bucketRequest(syncRules, 'global[]'))
-    );
+    expect(checksumAfter.get(request.bucket)).toEqual(checksumBefore.get(request.bucket));
+    expect(checksumAfter2.get(request.bucket)).toEqual(checksumBefore.get(request.bucket));
 
     test_utils.validateCompactedBucket(dataBefore, dataAfter);
   });
@@ -137,6 +126,7 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -177,37 +167,29 @@ bucket_definitions:
     });
 
     const checkpoint = result!.flushed_op;
+    const request = bucketRequest(syncRules, 'global[]');
 
-    const batchBefore = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
-    );
+    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
 
+    // op_id sequence depends on the storage implementation
     expect(dataBefore).toMatchObject([
       {
-        checksum: 2634521662,
         object_id: 't1',
-        op: 'PUT',
-        op_id: '1'
+        op: 'PUT'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
-        op: 'PUT',
-        op_id: '2'
+        op: 'PUT'
       },
       {
-        checksum: 4228978084,
         object_id: 't1',
-        op: 'REMOVE',
-        op_id: '3'
+        op: 'REMOVE'
       },
       {
-        checksum: 4243212114,
         object_id: 't2',
-        op: 'PUT',
-        op_id: '4'
+        op: 'PUT'
       }
     ]);
 
@@ -219,29 +201,28 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
-    );
+    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataAfter = batchAfter.chunkData.data;
     bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']));
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
 
-    expect(batchAfter.targetOp).toEqual(4n);
+    expect(batchAfter.targetOp).toBeLessThanOrEqual(checkpoint);
     expect(dataAfter).toMatchObject([
       {
-        checksum: -1778190028,
-        op: 'CLEAR',
-        op_id: '3'
+        checksum: addChecksums(
+          addChecksums(dataBefore[0].checksum as number, dataBefore[1].checksum as number),
+          dataBefore[2].checksum as number
+        ),
+        op: 'CLEAR'
       },
       {
-        checksum: 4243212114,
+        checksum: dataBefore[3].checksum,
         object_id: 't2',
-        op: 'PUT',
-        op_id: '4'
+        op: 'PUT'
       }
     ]);
-    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual({
-      ...checksumBefore.get(bucketRequest(syncRules, 'global[]')),
+    expect(checksumAfter.get(request.bucket)).toEqual({
+      ...checksumBefore.get(request.bucket),
       count: 2
     });
 
@@ -260,6 +241,7 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -291,7 +273,8 @@ bucket_definitions:
     });
 
     const checkpoint1 = result!.flushed_op;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint1, bucketRequests(syncRules, ['global[]']));
+    const request = bucketRequest(syncRules, 'global[]');
+    const checksumBefore = await bucketStorage.getChecksums(checkpoint1, [request]);
 
     const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       await batch.save({
@@ -314,25 +297,20 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]]))
-    );
+    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
     const dataAfter = batchAfter.chunkData.data;
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, bucketRequests(syncRules, ['global[]']));
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
 
-    expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
       {
-        checksum: 1874612650,
-        op: 'CLEAR',
-        op_id: '4'
+        op: 'CLEAR'
       }
     ]);
-    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual({
-      bucket: bucketRequest(syncRules, 'global[]'),
+    expect(checksumAfter.get(request.bucket)).toEqual({
+      bucket: request.bucket,
       count: 1,
-      checksum: 1874612650
+      checksum: dataAfter[0].checksum
     });
   });
 
@@ -351,6 +329,7 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       /**
        * Repeatedly create operations which fall into different buckets.
        * The bucket operations are purposely interleaved as the op_id increases.
@@ -477,7 +456,8 @@ bucket_definitions:
     );
     const bucketStorage = factory.getInstance(syncRules);
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -528,13 +508,17 @@ bucket_definitions:
       await batch.commit('2/1');
     });
     const checkpoint2 = result2!.flushed_op;
+    const request = bucketRequest(syncRules, 'global[]');
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, bucketRequests(syncRules, ['global[]']));
-    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toEqual({
-      bucket: bucketRequest(syncRules, 'global[]'),
-      count: 4,
-      checksum: 1874612650
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const globalChecksum = checksumAfter.get(request.bucket);
+    expect(globalChecksum).toMatchObject({
+      bucket: request.bucket,
+      count: 4
     });
+
+    // storage-specific checksum - just check that it does not change
+    expect(globalChecksum).toMatchSnapshot();
   });
 
   test('partial checksums after compacting (2)', async () => {
@@ -549,6 +533,7 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: storage.SaveOperationTag.INSERT,
@@ -593,12 +578,72 @@ bucket_definitions:
     });
 
     const checkpoint2 = result2!.flushed_op;
+    const request = bucketRequest(syncRules, 'global[]');
     // Check that the checksum was correctly updated with the clear operation after having a cached checksum
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, bucketRequests(syncRules, ['global[]']));
-    expect(checksumAfter.get(bucketRequest(syncRules, 'global[]'))).toMatchObject({
-      bucket: bucketRequest(syncRules, 'global[]'),
-      count: 1,
-      checksum: -1481659821
+    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const globalChecksum = checksumAfter.get(request.bucket);
+    expect(globalChecksum).toMatchObject({
+      bucket: request.bucket,
+      count: 1
+    });
+    // storage-specific checksum - just check that it does not change
+    expect(globalChecksum).toMatchSnapshot();
+  });
+
+  test('defaults maxOpId to current checkpoint', async () => {
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+      `)
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+
+    const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+      await batch.commit('1/1');
     });
+
+    const checkpoint1 = result1!.flushed_op;
+
+    const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      // This is flushed but not committed (does not advance the checkpoint)
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.UPDATE,
+        after: { id: 't1' },
+        afterReplicaId: test_utils.rid('t1')
+      });
+    });
+    const checkpoint2 = result2!.flushed_op;
+
+    const checkpointBeforeCompact = await bucketStorage.getCheckpoint();
+    expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1);
+
+    // With default options, Postgres compaction should use the active checkpoint.
+    await bucketStorage.compact({
+      moveBatchLimit: 1,
+      moveBatchQueryLimit: 1,
+      minBucketChanges: 1,
+      minChangeRatio: 0
+    });
+
+    const batchAfterDefaultCompact = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    );
+
+    // Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint.
+    expect(batchAfterDefaultCompact.chunkData.data).toMatchObject([
+      { op_id: '1', op: 'PUT', object_id: 't1' },
+      { op_id: '2', op: 'PUT', object_id: 't1' }
+    ]);
   });
 }

package/src/tests/register-data-storage-checkpoint-tests.ts

@@ -12,15 +12,24 @@ import * as test_utils from '../test-utils/test-utils-index.js';
  *
  * ```
  */
-export function registerDataStorageCheckpointTests(generateStorageFactory: storage.TestStorageFactory) {
+export function registerDataStorageCheckpointTests(config: storage.TestStorageConfig) {
+  const generateStorageFactory = config.factory;
+  const storageVersion = config.storageVersion;
+
   test('managed write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
-      updateSyncRulesFromYaml(`
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
-    `)
+    `,
+        {
+          validate: false,
+          storageVersion
+        }
+      )
     );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
 
@@ -30,6 +39,10 @@ bucket_definitions:
       .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
       [Symbol.asyncIterator]();
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
       heads: { '1': '5/0' },
       user_id: 'user1'
@@ -55,14 +68,24 @@ bucket_definitions:
   test('managed write checkpoints - write after checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
-      updateSyncRulesFromYaml(`
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
-    `)
+    `,
+        {
+          validate: false,
+          storageVersion
+        }
+      )
     );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
@@ -116,15 +139,25 @@ bucket_definitions:
   test('custom write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
-      updateSyncRulesFromYaml(`
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
-    `)
+    `,
+        {
+          validate: false,
+          storageVersion
+        }
+      )
     );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
@@ -155,15 +188,25 @@ bucket_definitions:
   test('custom write checkpoints - standalone checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
-      updateSyncRulesFromYaml(`
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
-    `)
+    `,
+        {
+          validate: false,
+          storageVersion
+        }
+      )
     );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
@@ -197,15 +240,25 @@ bucket_definitions:
   test('custom write checkpoints - write after checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
-      updateSyncRulesFromYaml(`
+      updateSyncRulesFromYaml(
+        `
 bucket_definitions:
   mybucket:
     data: []
-    `)
+    `,
+        {
+          validate: false,
+          storageVersion
+        }
+      )
     );
     const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
     bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
 
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.markAllSnapshotDone('1/1');
+    });
+
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage