@powersync/service-module-mongodb 0.13.2 → 0.15.0

package/test/src/change_stream.test.ts

@@ -3,12 +3,11 @@ import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test, vi } from 'vitest';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { storage } from '@powersync/service-core';
 import { test_utils } from '@powersync/service-core-tests';
 
 import { PostImagesOption } from '@module/types/types.js';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
-import { describeWithStorage } from './util.js';
+import { describeWithStorage, StorageVersionTestContext } from './util.js';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:
@@ -21,9 +20,12 @@ describe('change stream', () => {
   describeWithStorage({ timeout: 20_000 }, defineChangeStreamTests);
 });
 
-function defineChangeStreamTests(factory: storage.TestStorageFactory) {
+function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
+    return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
+  };
   test('replicating basic values', async () => {
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.READ_ONLY }
     });
     const { db } = context;
@@ -59,7 +61,7 @@ bucket_definitions:
   });
 
   test('replicating wildcard', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -91,7 +93,7 @@ bucket_definitions:
   });
 
   test('updateLookup - no fullDocument available', async () => {
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.OFF }
     });
     const { db, client } = context;
@@ -137,7 +139,7 @@ bucket_definitions:
   test('postImages - autoConfigure', async () => {
     // Similar to the above test, but with postImages enabled.
     // This resolves the consistency issue.
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
     });
     const { db, client } = context;
@@ -185,7 +187,7 @@ bucket_definitions:
   test('postImages - on', async () => {
     // Similar to postImages - autoConfigure, but does not auto-configure.
     // changeStreamPreAndPostImages must be manually configured.
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.READ_ONLY }
     });
     const { db, client } = context;
@@ -230,7 +232,7 @@ bucket_definitions:
   });
 
   test('replicating case sensitive table', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(`
       bucket_definitions:
@@ -254,7 +256,7 @@ bucket_definitions:
   });
 
   test('replicating large values', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(`
       bucket_definitions:
@@ -285,7 +287,7 @@ bucket_definitions:
   });
 
   test('replicating dropCollection', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     const syncRuleContent = `
 bucket_definitions:
@@ -317,7 +319,7 @@ bucket_definitions:
   });
 
   test('replicating renameCollection', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     const syncRuleContent = `
 bucket_definitions:
@@ -348,7 +350,7 @@ bucket_definitions:
   });
 
   test('initial sync', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
@@ -373,7 +375,7 @@ bucket_definitions:
     // MongoServerError: PlanExecutor error during aggregation :: caused by :: BSONObj size: 33554925 (0x20001ED) is invalid.
     // Size must be between 0 and 16793600(16MB)
 
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     await context.updateSyncRules(`bucket_definitions:
       global:
         data:
@@ -422,7 +424,7 @@ bucket_definitions:
   });
 
   test('collection not in sync rules', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
@@ -439,7 +441,7 @@ bucket_definitions:
   });
 
   test('postImages - new collection with postImages enabled', async () => {
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
     });
     const { db } = context;
@@ -472,7 +474,7 @@ bucket_definitions:
   });
 
   test('postImages - new collection with postImages disabled', async () => {
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
     });
     const { db } = context;
@@ -502,7 +504,7 @@ bucket_definitions:
   });
 
   test('recover from error', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(`
 bucket_definitions:

package/test/src/change_stream_utils.ts

@@ -4,13 +4,18 @@ import {
   createCoreReplicationMetrics,
   initializeCoreReplicationMetrics,
   InternalOpId,
+  LEGACY_STORAGE_VERSION,
   OplogEntry,
   ProtocolOpId,
   ReplicationCheckpoint,
+  storage,
+  STORAGE_VERSION_CONFIG,
   SyncRulesBucketStorage,
-  TestStorageOptions
+  TestStorageOptions,
+  updateSyncRulesFromYaml,
+  utils
 } from '@powersync/service-core';
-import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 
 import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
 import { MongoManager } from '@module/replication/MongoManager.js';
@@ -22,7 +27,9 @@ import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
   private abortController = new AbortController();
-  private streamPromise?: Promise<void>;
+  private streamPromise?: Promise<PromiseSettledResult<void>>;
+  private syncRulesId?: number;
+  private syncRulesContent?: storage.PersistedSyncRulesContent;
   public storage?: SyncRulesBucketStorage;
 
   /**
@@ -35,6 +42,7 @@ export class ChangeStreamTestContext {
     factory: (options: TestStorageOptions) => Promise<BucketStorageFactory>,
     options?: {
       doNotClear?: boolean;
+      storageVersion?: number;
       mongoOptions?: Partial<NormalizedMongoConnectionConfig>;
       streamOptions?: Partial<ChangeStreamOptions>;
     }
@@ -45,13 +53,19 @@ export class ChangeStreamTestContext {
     if (!options?.doNotClear) {
       await clearTestDb(connectionManager.db);
     }
-    return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions);
+
+    const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION;
+    const versionedBuckets = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false;
+
+    return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion, versionedBuckets);
   }
 
   constructor(
     public factory: BucketStorageFactory,
     public connectionManager: MongoManager,
-    private streamOptions?: Partial<ChangeStreamOptions>
+    private streamOptions: Partial<ChangeStreamOptions> = {},
+    private storageVersion: number = LEGACY_STORAGE_VERSION,
+    private versionedBuckets: boolean = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false
   ) {
     createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
     initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
@@ -88,7 +102,11 @@ export class ChangeStreamTestContext {
   }
 
   async updateSyncRules(content: string) {
-    const syncRules = await this.factory.updateSyncRules({ content: content, validate: true });
+    const syncRules = await this.factory.updateSyncRules(
+      updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
+    );
+    this.syncRulesId = syncRules.id;
+    this.syncRulesContent = syncRules;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
@@ -99,11 +117,20 @@ export class ChangeStreamTestContext {
       throw new Error(`Next sync rules not available`);
     }
 
+    this.syncRulesId = syncRules.id;
+    this.syncRulesContent = syncRules;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
 
-  get walStream() {
+  private getSyncRulesContent(): storage.PersistedSyncRulesContent {
+    if (this.syncRulesContent == null) {
+      throw new Error('Sync rules not configured - call updateSyncRules() first');
+    }
+    return this.syncRulesContent;
+  }
+
+  get streamer() {
     if (this.storage == null) {
       throw new Error('updateSyncRules() first');
     }
@@ -125,7 +152,7 @@ export class ChangeStreamTestContext {
   }
 
   async replicateSnapshot() {
-    await this.walStream.initReplication();
+    await this.streamer.initReplication();
   }
 
   /**
@@ -143,13 +170,21 @@ export class ChangeStreamTestContext {
   }
 
   startStreaming() {
-    return (this.streamPromise = this.walStream.streamChanges());
+    this.streamPromise = this.streamer
+      .streamChanges()
+      .then(() => ({ status: 'fulfilled', value: undefined }) satisfies PromiseFulfilledResult<void>)
+      .catch((reason) => ({ status: 'rejected', reason }) satisfies PromiseRejectedResult);
+    return this.streamPromise;
   }
 
   async getCheckpoint(options?: { timeout?: number }) {
     let checkpoint = await Promise.race([
       getClientCheckpoint(this.client, this.db, this.factory, { timeout: options?.timeout ?? 15_000 }),
-      this.streamPromise
+      this.streamPromise?.then((e) => {
+        if (e.status == 'rejected') {
+          throw e.reason;
+        }
+      })
     ]);
     if (checkpoint == null) {
       // This indicates an issue with the test setup - streamingPromise completed instead
@@ -161,7 +196,8 @@ export class ChangeStreamTestContext {
 
   async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, InternalOpId>(Object.entries(buckets));
+    const syncRules = this.getSyncRulesContent();
+    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start));
     return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
@@ -170,8 +206,9 @@ export class ChangeStreamTestContext {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
+    const syncRules = this.getSyncRulesContent();
     const checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, InternalOpId>([[bucket, start]]);
+    let map = [bucketRequest(syncRules, bucket, start)];
     let data: OplogEntry[] = [];
     while (true) {
       const batch = this.storage!.getBucketDataBatch(checkpoint, map);
@@ -181,20 +218,27 @@ export class ChangeStreamTestContext {
       if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
         break;
       }
-      map.set(bucket, BigInt(batches[0]!.chunkData.next_after));
+      map = [bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))];
     }
     return data;
   }
 
-  async getChecksums(buckets: string[], options?: { timeout?: number }) {
+  async getChecksums(buckets: string[], options?: { timeout?: number }): Promise<utils.ChecksumMap> {
     let checkpoint = await this.getCheckpoint(options);
-    return this.storage!.getChecksums(checkpoint, buckets);
+    const syncRules = this.getSyncRulesContent();
+    const versionedBuckets = buckets.map((bucket) => bucketRequest(syncRules, bucket, 0n));
+    const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
+
+    const unversioned: utils.ChecksumMap = new Map();
+    for (let i = 0; i < buckets.length; i++) {
+      unversioned.set(buckets[i], checksums.get(versionedBuckets[i].bucket)!);
+    }
+    return unversioned;
   }
 
   async getChecksum(bucket: string, options?: { timeout?: number }) {
-    let checkpoint = await this.getCheckpoint(options);
-    const map = await this.storage!.getChecksums(checkpoint, [bucket]);
-    return map.get(bucket);
+    const checksums = await this.getChecksums([bucket], options);
+    return checksums.get(bucket);
   }
 }
 

package/test/src/chunked_snapshot.test.ts

@@ -1,18 +1,22 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { reduceBucket, TestStorageFactory } from '@powersync/service-core';
+import { reduceBucket } from '@powersync/service-core';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
-import { describeWithStorage } from './util.js';
+import { describeWithStorage, StorageVersionTestContext } from './util.js';
 
 describe('chunked snapshots', () => {
   describeWithStorage({ timeout: 120_000 }, defineBatchTests);
 });
 
-function defineBatchTests(factory: TestStorageFactory) {
+function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
+    return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
+  };
+
   // This is not as sensitive to the id type as postgres, but we still test a couple of cases
   test('chunked snapshot (int32)', async () => {
     await testChunkedSnapshot({
@@ -93,7 +97,7 @@ function defineBatchTests(factory: TestStorageFactory) {
     const idToSqlite = options.idToSqlite ?? ((n) => n);
     const idToString = (id: any) => String(idToSqlite(id));
 
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openContext({
       // We need to use a smaller chunk size here, so that we can run a query in between chunks
       streamOptions: { snapshotChunkLength: 100 }
     });

package/test/src/resume.test.ts

@@ -1,19 +1,22 @@
 import { ChangeStreamInvalidatedError } from '@module/replication/ChangeStream.js';
 import { MongoManager } from '@module/replication/MongoManager.js';
 import { normalizeConnectionConfig } from '@module/types/types.js';
-import { BucketStorageFactory, TestStorageOptions } from '@powersync/service-core';
 import { describe, expect, test } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
 import { env } from './env.js';
-import { describeWithStorage } from './util.js';
+import { describeWithStorage, StorageVersionTestContext } from './util.js';
 
 describe('mongodb resuming replication', () => {
   describeWithStorage({}, defineResumeTest);
 });
 
-function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Promise<BucketStorageFactory>) {
+function defineResumeTest({ factory: factoryGenerator, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
+    return ChangeStreamTestContext.open(factoryGenerator, { ...options, storageVersion });
+  };
+
   test('resuming with a different source database', async () => {
-    await using context = await ChangeStreamTestContext.open(factoryGenerator);
+    await using context = await openContext();
     const { db } = context;
 
     await context.updateSyncRules(/* yaml */
@@ -53,13 +56,14 @@ function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Pr
     const factory = await factoryGenerator({ doNotClear: true });
 
     // Create a new context without updating the sync rules
-    await using context2 = new ChangeStreamTestContext(factory, connectionManager);
+    await using context2 = new ChangeStreamTestContext(factory, connectionManager, {}, storageVersion);
     const activeContent = await factory.getActiveSyncRulesContent();
     context2.storage = factory.getInstance(activeContent!);
 
     // If this test times out, it likely didn't throw the expected error here.
-    const error = await context2.startStreaming().catch((ex) => ex);
+    const result = await context2.startStreaming();
     // The ChangeStreamReplicationJob will detect this and throw a ChangeStreamInvalidatedError
-    expect(error).toBeInstanceOf(ChangeStreamInvalidatedError);
+    expect(result.status).toEqual('rejected');
+    expect((result as PromiseRejectedResult).reason).toBeInstanceOf(ChangeStreamInvalidatedError);
   });
 }

package/test/src/resuming_snapshots.test.ts

@@ -8,19 +8,19 @@ import { env } from './env.js';
 import { describeWithStorage } from './util.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
-  describeWithStorage({ timeout: 240_000 }, function (factory) {
+  describeWithStorage({ timeout: 240_000 }, function ({ factory, storageVersion }) {
     test('resuming initial replication (1)', async () => {
       // Stop early - likely to not include deleted row in first replication attempt.
-      await testResumingReplication(factory, 2000);
+      await testResumingReplication(factory, storageVersion, 2000);
     });
     test('resuming initial replication (2)', async () => {
       // Stop late - likely to include deleted row in first replication attempt.
-      await testResumingReplication(factory, 8000);
+      await testResumingReplication(factory, storageVersion, 8000);
     });
   });
 });
 
-async function testResumingReplication(factory: TestStorageFactory, stopAfter: number) {
+async function testResumingReplication(factory: TestStorageFactory, storageVersion: number, stopAfter: number) {
   // This tests interrupting and then resuming initial replication.
   // We interrupt replication after test_data1 has fully replicated, and
   // test_data2 has partially replicated.
@@ -35,7 +35,10 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   let startRowCount: number;
 
   {
-    await using context = await ChangeStreamTestContext.open(factory, { streamOptions: { snapshotChunkLength: 1000 } });
+    await using context = await ChangeStreamTestContext.open(factory, {
+      storageVersion,
+      streamOptions: { snapshotChunkLength: 1000 }
+    });
 
     await context.updateSyncRules(`bucket_definitions:
   global:
@@ -87,6 +90,7 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
     // Bypass the usual "clear db on factory open" step.
     await using context2 = await ChangeStreamTestContext.open(factory, {
       doNotClear: true,
+      storageVersion,
       streamOptions: { snapshotChunkLength: 1000 }
     });
 

package/test/src/slow_tests.test.ts

@@ -2,19 +2,21 @@ import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { storage } from '@powersync/service-core';
-
 import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js';
 import { env } from './env.js';
-import { describeWithStorage } from './util.js';
+import { describeWithStorage, StorageVersionTestContext } from './util.js';
 
 describe.runIf(env.CI || env.SLOW_TESTS)('change stream slow tests', { timeout: 60_000 }, function () {
   describeWithStorage({}, defineSlowTests);
 });
 
-function defineSlowTests(factory: storage.TestStorageFactory) {
+function defineSlowTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
+    return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
+  };
+
   test('replicating snapshot with lots of data', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     // Test with low minSnapshotHistoryWindowInSeconds, to trigger:
     // > Read timestamp .. is older than the oldest available timestamp.
     // This happened when we had {snapshot: true} in the initial
@@ -52,7 +54,7 @@ bucket_definitions:
     // changestream), we may miss updates, which this test would
     // hopefully catch.
 
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(`
 bucket_definitions:

package/test/src/util.ts

@@ -3,9 +3,14 @@ import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 
 import * as types from '@module/types/types.js';
-import { env } from './env.js';
-import { BSON_DESERIALIZE_DATA_OPTIONS, TestStorageFactory } from '@powersync/service-core';
+import {
+  BSON_DESERIALIZE_DATA_OPTIONS,
+  SUPPORTED_STORAGE_VERSIONS,
+  TestStorageConfig,
+  TestStorageFactory
+} from '@powersync/service-core';
 import { describe, TestOptions } from 'vitest';
+import { env } from './env.js';
 
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
 
@@ -19,18 +24,38 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoT
   isCI: env.CI
 });
 
-export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestStorageFactoryGenerator({
+export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestSetup({
   url: env.PG_STORAGE_TEST_URL
 });
 
-export function describeWithStorage(options: TestOptions, fn: (factory: TestStorageFactory) => void) {
-  describe.skipIf(!env.TEST_MONGO_STORAGE)(`mongodb storage`, options, function () {
-    fn(INITIALIZED_MONGO_STORAGE_FACTORY);
-  });
+export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;
 
-  describe.skipIf(!env.TEST_POSTGRES_STORAGE)(`postgres storage`, options, function () {
-    fn(INITIALIZED_POSTGRES_STORAGE_FACTORY);
-  });
+export interface StorageVersionTestContext {
+  factory: TestStorageFactory;
+  storageVersion: number;
+}
+
+export function describeWithStorage(options: TestOptions, fn: (context: StorageVersionTestContext) => void) {
+  const describeFactory = (storageName: string, config: TestStorageConfig) => {
+    describe(`${storageName} storage`, options, function () {
+      for (const storageVersion of TEST_STORAGE_VERSIONS) {
+        describe(`storage v${storageVersion}`, function () {
+          fn({
+            factory: config.factory,
+            storageVersion
+          });
+        });
+      }
+    });
+  };
+
+  if (env.TEST_MONGO_STORAGE) {
+    describeFactory('mongodb', INITIALIZED_MONGO_STORAGE_FACTORY);
+  }
+
+  if (env.TEST_POSTGRES_STORAGE) {
+    describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY);
+  }
 }
 
 export async function clearTestDb(db: mongo.Db) {