@powersync/service-core-tests 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

package/src/test-utils/leased-storage.ts

@@ -0,0 +1,58 @@
+import { storage } from '@powersync/service-core';
+
+const leases = new WeakMap<storage.BucketStorageFactory, Map<number, Promise<storage.ReplicationLock>>>();
+
+/**
+ * Acquire a real replication lease before constructing writable test storage.
+ * Writers in the same test job share a lease per factory/stream. The factory owns
+ * these leases and releases them at teardown; takeover tests manage their own leases.
+ */
+export async function getTestStorage<T extends storage.BucketStorageFactory>(
+  factory: T,
+  stream: storage.PersistedReplicationStream,
+  options?: Parameters<T['getInstance']>[1]
+): Promise<ReturnType<T['getInstance']>> {
+  let owned = leases.get(factory);
+  if (owned == null) {
+    owned = new Map();
+    leases.set(factory, owned);
+    const factoryLeases = owned;
+    // `await using` captures the disposer before this helper runs. Register a
+    // lifecycle callback instead of replacing that already-captured method.
+    const unregister = factory.registerListener({
+      beforeDispose: async () => {
+        unregister();
+        try {
+          for (const lock of factoryLeases.values()) {
+            await (await lock).release();
+          }
+        } finally {
+          factoryLeases.clear();
+        }
+      }
+    });
+  }
+  let pending = owned.get(stream.replicationStreamId);
+  if (pending == null) {
+    pending = stream.lock();
+    owned.set(stream.replicationStreamId, pending);
+  }
+  let lock: storage.ReplicationLock;
+  try {
+    lock = await pending;
+  } catch (error) {
+    owned.delete(stream.replicationStreamId);
+    throw error;
+  }
+  return factory.getInstance(stream, { ...options, replicationLock: lock }) as ReturnType<T['getInstance']>;
+}
+
+/** End a test job before starting a replacement through another factory. */
+export async function releaseTestStorageLease(factory: storage.BucketStorageFactory, streamId: number) {
+  const owned = leases.get(factory);
+  const pending = owned?.get(streamId);
+  if (pending != null) {
+    await (await pending).release();
+    owned!.delete(streamId);
+  }
+}

package/src/test-utils/storage-combinations.ts

@@ -0,0 +1,51 @@
+import {
+  STORAGE_VERSION_1,
+  STORAGE_VERSION_2,
+  STORAGE_VERSION_4,
+  TestStorageConfig,
+  TestStorageFactory
+} from '@powersync/service-core';
+import { describe, TestOptions } from 'vitest';
+
+const MONGO_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_4];
+const POSTGRES_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2];
+
+export interface StorageVersionTestContext {
+  factory: TestStorageFactory;
+  storageVersion: number;
+}
+
+export interface StorageCombinationTestConfig {
+  mongodb?: TestStorageConfig;
+  postgres?: TestStorageConfig;
+}
+
+export function describeStorageCombinations(
+  config: StorageCombinationTestConfig,
+  options: TestOptions & { storageVersions?: number[] },
+  fn: (context: StorageVersionTestContext) => void
+) {
+  const describeFactory = (storageName: string, storage: TestStorageConfig, supportedStorageVersions: number[]) => {
+    const storageVersions = (options.storageVersions ?? supportedStorageVersions).filter((version) =>
+      supportedStorageVersions.includes(version)
+    );
+    describe(`${storageName} storage`, options, function () {
+      for (const storageVersion of storageVersions) {
+        describe(`storage v${storageVersion}`, function () {
+          fn({
+            factory: storage.factory,
+            storageVersion
+          });
+        });
+      }
+    });
+  };
+
+  if (config.mongodb) {
+    describeFactory('mongodb', config.mongodb, MONGO_STORAGE_VERSIONS);
+  }
+
+  if (config.postgres) {
+    describeFactory('postgres', config.postgres, POSTGRES_STORAGE_VERSIONS);
+  }
+}

package/src/test-utils/test-utils-index.ts

@@ -1,6 +1,8 @@
 export * from './AbstractStreamTestContext.js';
 export * from './bucket-validation.js';
 export * from './general-utils.js';
+export * from './leased-storage.js';
 export * from './MetricsHelper.js';
+export * from './storage-combinations.js';
 export * from './StorageDataHelpers.js';
 export * from './stream_utils.js';

package/src/tests/register-compacting-tests.ts

@@ -1,8 +1,9 @@
 import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { expect, test } from 'vitest';
+import { getTestStorage } from '../test-utils/leased-storage.js';
 import * as test_utils from '../test-utils/test-utils-index.js';
 import { bucketRequest } from '../test-utils/test-utils-index.js';
-import { bucketRequestMap, bucketRequests } from './util.js';
+import { bucketRequestMap, bucketRequests, compactActive } from './util.js';
 
 export function registerCompactTests(config: storage.TestStorageConfig) {
   const generateStorageFactory = config.factory;
@@ -16,7 +17,7 @@ bucket_definitions:
     data: [select * from test]
     `)
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = await getTestStorage(factory, syncRules);
     const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -81,7 +82,7 @@ bucket_definitions:
     ]);
     expect(batchBefore.targetOp).toEqual(null);
 
-    await bucketStorage.compact({
+    await compactActive(factory, {
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
@@ -128,7 +129,7 @@ bucket_definitions:
     data: [select * from test]
     `)
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = await getTestStorage(factory, syncRules);
     const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -202,7 +203,7 @@ bucket_definitions:
       }
     ]);
 
-    await bucketStorage.compact({
+    await compactActive(factory, {
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
@@ -249,7 +250,7 @@ bucket_definitions:
     data: [select * from test]
     `)
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = await getTestStorage(factory, syncRules);
     const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -301,7 +302,7 @@ bucket_definitions:
     await writer2.flush();
     const checkpoint2 = writer2.last_flushed_op!;
 
-    await bucketStorage.compact({
+    await compactActive(factory, {
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
@@ -340,7 +341,7 @@ bucket_definitions:
             data:
               - select * from test where b = bucket.b`)
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = await getTestStorage(factory, syncRules);
     const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -416,7 +417,7 @@ bucket_definitions:
 
     const checkpoint = writer.last_flushed_op!;
 
-    await bucketStorage.compact({
+    await compactActive(factory, {
       clearBatchLimit: 100,
       moveBatchLimit: 100,
       moveBatchQueryLimit: 100, // Larger limit for a larger window of operations
@@ -471,7 +472,7 @@ bucket_definitions:
     data: [select * from test]
     `)
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = await getTestStorage(factory, syncRules);
     const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -507,7 +508,7 @@ bucket_definitions:
     await writer.commit('1/1');
     await writer.flush();
 
-    await bucketStorage.compact({
+    await compactActive(factory, {
       clearBatchLimit: 2,
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
@@ -550,7 +551,7 @@ bucket_definitions:
     data: [select * from test]
     `)
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = await getTestStorage(factory, syncRules);
     const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -595,7 +596,7 @@ bucket_definitions:
     await writer2.commit('2/1');
     await writer2.flush();
 
-    await bucketStorage.compact({
+    await compactActive(factory, {
       clearBatchLimit: 20,
       moveBatchLimit: 10,
       moveBatchQueryLimit: 10,
@@ -625,7 +626,7 @@ bucket_definitions:
     data: [select * from test]
       `)
     );
-    const bucketStorage = factory.getInstance(syncRules);
+    const bucketStorage = await getTestStorage(factory, syncRules);
     const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -658,7 +659,7 @@ bucket_definitions:
     expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1);
 
     // With default options, Postgres compaction should use the active checkpoint.
-    await bucketStorage.compact({
+    await compactActive(factory, {
       moveBatchLimit: 1,
       moveBatchQueryLimit: 1,
       minBucketChanges: 1,

package/src/tests/register-data-storage-checkpoint-tests.ts

@@ -1,6 +1,8 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { expect, test } from 'vitest';
+import { getTestStorage } from '../test-utils/leased-storage.js';
 import * as test_utils from '../test-utils/test-utils-index.js';
+import { compactActive } from './util.js';
 
 /**
  * @example
@@ -31,12 +33,15 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!);
 
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
-      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      .watchCheckpointChanges({
+        user_id: 'user1',
+        signal: abortController.signal
+      })
       [Symbol.asyncIterator]();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -77,7 +82,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await writer.markAllSnapshotDone('1/1');
@@ -85,7 +90,10 @@ bucket_definitions:
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
-      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      .watchCheckpointChanges({
+        user_id: 'user1',
+        signal: abortController.signal
+      })
       [Symbol.asyncIterator]();
 
     await writer.keepalive('5/0');
@@ -143,7 +151,7 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!);
 
     const first = await createManagedWriteCheckpointResult(bucketStorage, {
       user_id: 'user1',
@@ -194,7 +202,7 @@ bucket_definitions:
     expect(generated.shouldAdvance).toBe(true);
   });
 
-  test('custom write checkpoints - checkpoint after write', async (context) => {
+  test.runIf((storageVersion ?? 1) < 3)('custom write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
       updateSyncRulesFromYaml(
@@ -209,8 +217,10 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+    const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!);
+    bucketStorage.setWriteCheckpointMode({
+      mode: storage.WriteCheckpointMode.CUSTOM
+    });
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await writer.markAllSnapshotDone('1/1');
@@ -218,7 +228,10 @@ bucket_definitions:
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
-      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      .watchCheckpointChanges({
+        user_id: 'user1',
+        signal: abortController.signal
+      })
       [Symbol.asyncIterator]();
 
     writer.addCustomWriteCheckpoint({
@@ -240,7 +253,7 @@ bucket_definitions:
     });
   });
 
-  test('custom write checkpoints - standalone checkpoint', async (context) => {
+  test.runIf((storageVersion ?? 1) < 3)('custom write checkpoints - standalone checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
       updateSyncRulesFromYaml(
@@ -255,8 +268,10 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+    const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!);
+    bucketStorage.setWriteCheckpointMode({
+      mode: storage.WriteCheckpointMode.CUSTOM
+    });
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await writer.markAllSnapshotDone('1/1');
@@ -264,7 +279,10 @@ bucket_definitions:
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
-      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      .watchCheckpointChanges({
+        user_id: 'user1',
+        signal: abortController.signal
+      })
       [Symbol.asyncIterator]();
 
     // Flush to clear state
@@ -289,7 +307,7 @@ bucket_definitions:
     });
   });
 
-  test('custom write checkpoints - write after checkpoint', async (context) => {
+  test.runIf((storageVersion ?? 1) < 3)('custom write checkpoints - write after checkpoint', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
       updateSyncRulesFromYaml(
@@ -304,8 +322,10 @@ bucket_definitions:
         }
       )
     );
-    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+    const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!);
+    bucketStorage.setWriteCheckpointMode({
+      mode: storage.WriteCheckpointMode.CUSTOM
+    });
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await writer.markAllSnapshotDone('1/1');
@@ -313,7 +333,10 @@ bucket_definitions:
     const abortController = new AbortController();
     context.onTestFinished(() => abortController.abort());
     const iter = bucketStorage
-      .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal })
+      .watchCheckpointChanges({
+        user_id: 'user1',
+        signal: abortController.signal
+      })
       [Symbol.asyncIterator]();
 
     await writer.keepalive('5/0');
@@ -368,49 +391,55 @@ bucket_definitions:
     });
   });
 
-  test('custom write checkpoints - checkpoint request markers are temporary', async () => {
-    await using factory = await generateStorageFactory();
-    const r = await factory.configureSyncRules(
-      updateSyncRulesFromYaml(
-        `
+  test.runIf((storageVersion ?? 1) < 3)(
+    'custom write checkpoints - checkpoint request markers are temporary',
+    async () => {
+      await using factory = await generateStorageFactory();
+      const r = await factory.configureSyncRules(
+        updateSyncRulesFromYaml(
+          `
 bucket_definitions:
   mybucket:
     data: []
     `,
-        {
-          validate: false,
-          storageVersion
-        }
-      )
-    );
-    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
-    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
-
-    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-    await writer.markAllSnapshotDone('1/1');
-
-    writer.addCustomWriteCheckpoint({
-      checkpoint: 5n,
-      user_id: 'persistent'
-    });
-    writer.addCustomWriteCheckpoint({
-      checkpoint: 6n,
-      user_id: 'temporary',
-      checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z')
-    });
-    await writer.flush();
-
-    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
-    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toEqual(6n);
-
-    await bucketStorage.compact({
-      compactBuckets: [],
-      deleteCheckpointRequestsBefore: new Date(Date.now() + 1_000)
-    });
-
-    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
-    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toBeNull();
-  });
+          {
+            validate: false,
+            storageVersion
+          }
+        )
+      );
+      const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!);
+      bucketStorage.setWriteCheckpointMode({
+        mode: storage.WriteCheckpointMode.CUSTOM
+      });
+
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      await writer.markAllSnapshotDone('1/1');
+
+      writer.addCustomWriteCheckpoint({
+        checkpoint: 5n,
+        user_id: 'persistent'
+      });
+      writer.addCustomWriteCheckpoint({
+        checkpoint: 6n,
+        user_id: 'temporary',
+        checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z')
+      });
+      await writer.flush();
+      await writer.keepalive('1/1');
+
+      await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
+      await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toEqual(6n);
+
+      await compactActive(factory, {
+        compactBuckets: [],
+        deleteCheckpointRequestsBefore: new Date(Date.now() + 1_000)
+      });
+
+      await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
+      await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toBeNull();
+    }
+  );
 }
 
 async function createManagedWriteCheckpoint(