@powersync/service-module-mongodb 0.17.0 → 0.18.0

package/src/replication/ChangeStream.ts

@@ -116,7 +116,7 @@ export class ChangeStream {
   constructor(options: ChangeStreamOptions) {
     this.storage = options.storage;
     this.metrics = options.metrics;
-    this.group_id = options.storage.group_id;
+    this.group_id = options.storage.replicationStreamId;
     this.connections = options.connections;
     this.maxAwaitTimeMS = options.maxAwaitTimeMS ?? 10_000;
     this.snapshotChunkLength = options.snapshotChunkLength ?? 6_000;
@@ -132,11 +132,16 @@ export class ChangeStream {
     // so we use 90% of the socket timeout value.
     this.changeStreamTimeout = Math.ceil(this.client.options.socketTimeoutMS * 0.9);
 
-    this.logger = options.logger ?? this.storage.logger;
+    const baseLogger = options.logger ?? this.storage.logger;
+    // Unfortunately the Winston APIs don't have a nice way to append to the prefix,
+    // so we replace it here.
+    this.logger = baseLogger.child({ prefix: `[${this.storage.replicationStreamName}] [stream] ` });
+    const snapshotLogger = baseLogger.child({ prefix: `[${this.storage.replicationStreamName}] [snapshot] ` });
+
     this.snapshotter = new MongoSnapshotter({
       ...options,
       abortSignal: this.abortSignal,
-      logger: this.logger,
+      logger: snapshotLogger,
       checkpointStreamId: this.checkpointStreamId
     });
 
@@ -366,6 +371,7 @@ export class ChangeStream {
   async replicate() {
     let streamPromise: Promise<void> | null = null;
     let loopPromise: Promise<void> | null = null;
+    let cleanupPromise: Promise<void> | null = null;
     try {
       // If anything errors here, the entire replication process is halted, and
       // all connections automatically closed, including this one.
@@ -383,6 +389,11 @@ export class ChangeStream {
       if (!this.snapshotter.supportsConcurrentSnapshots) {
         await Promise.race([this.snapshotter.waitForInitialSnapshot(), loopPromise]);
       }
+      // Unlike the other two, this resolves on completion, not an indefinite loop.
+      cleanupPromise = this.cleanupStoppedSyncConfigs().catch((e) => {
+        this.abortController.abort(e);
+        throw e;
+      });
       streamPromise = this.streamChanges()
         .then(() => {
           throw new ReplicationAssertionError(`Replication stream exited unexpectedly`);
@@ -392,7 +403,7 @@ export class ChangeStream {
           throw e;
         });
 
-      const results = await Promise.allSettled([loopPromise, streamPromise]);
+      const results = await Promise.allSettled([loopPromise, streamPromise, cleanupPromise]);
       throw replicationLoopError(results);
     } catch (e) {
       await this.storage.reportError(e);
@@ -422,6 +433,15 @@ export class ChangeStream {
     }
   }
 
+  private async cleanupStoppedSyncConfigs() {
+    await this.storage.cleanupStoppedSyncConfigs?.({
+      signal: this.abortSignal,
+      logger: this.logger,
+      defaultSchema: this.defaultDb.databaseName,
+      sourceConnectionTag: this.connections.connectionTag
+    });
+  }
+
   private async streamChanges() {
     try {
       await this.streamChangesInternal();
@@ -733,25 +753,15 @@ export class ChangeStream {
                 timestamp: changeDocument.clusterTime!,
                 resume_token: changeDocument._id
               });
-              if (batch.lastCheckpointLsn != null && lsn < batch.lastCheckpointLsn) {
-                // Checkpoint out of order - should never happen with MongoDB.
-                // If it does happen, we throw an error to stop the replication - restarting should recover.
-                // Since we use batch.lastCheckpointLsn for the next resumeAfter, this should not result in an infinite loop.
-                // Originally a workaround for https://jira.mongodb.org/browse/NODE-7042.
-                // This has been fixed in the driver in the meantime, but we still keep this as a safety-check.
-                throw new ReplicationAssertionError(
-                  `Change resumeToken ${(changeDocument._id as any)._data} (${timestampToDate(changeDocument.clusterTime!).toISOString()}) is less than last checkpoint LSN ${batch.lastCheckpointLsn}. Restarting replication.`
-                );
-              }
 
               if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
                 waitForCheckpointLsn = null;
               }
-              const { checkpointBlocked } = await batch.commit(lsn, {
+              const { checkpointBlocked, checkpointCreated } = await batch.commit(lsn, {
                 oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
               });
 
-              if (!checkpointBlocked) {
+              if (!checkpointBlocked || checkpointCreated) {
                 this.replicationLag.markCommitted();
               }
             } else if (

package/src/replication/ChangeStreamReplicationJob.ts

@@ -47,7 +47,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
 
       if (e instanceof ChangeStreamInvalidatedError) {
         // This stops replication and restarts with a new instance
-        await this.options.storage.factory.restartReplication(this.storage.group_id);
+        await this.options.storage.factory.restartReplication(this.storage.replicationStreamId);
       }
 
       // No need to rethrow - the error is already logged, and retry behavior is the same on error

package/src/replication/ChangeStreamReplicator.ts

@@ -17,7 +17,7 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator<Chang
 
   createJob(options: replication.CreateJobOptions): ChangeStreamReplicationJob {
     return new ChangeStreamReplicationJob({
-      id: this.createJobId(options.storage.group_id),
+      id: this.createJobId(options.storage.replicationStreamId),
       storage: options.storage,
       metrics: this.metrics,
       connectionFactory: this.connectionFactory,

package/src/replication/MongoSnapshotter.ts

@@ -1,7 +1,6 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { container, ErrorCode, Logger, ReplicationAbortedError, ServiceError } from '@powersync/lib-services-framework';
 import {
-  InternalOpId,
   MetricsEngine,
   PerformanceTracer,
   SaveOperationTag,
@@ -71,7 +70,6 @@ export class MongoSnapshotter {
   private readonly queue = new Set<SnapshotQueueItem>();
   private initialSnapshotDone = Promise.withResolvers<void>();
   private nextItemQueued: PromiseWithResolvers<void> | null = null;
-  private lastSnapshotOpId: InternalOpId | null = null;
   private lastTouchedAt = performance.now();
 
   constructor(options: MongoSnapshotterOptions) {
@@ -112,12 +110,12 @@ export class MongoSnapshotter {
 
   async checkSlot(): Promise<InitResult> {
     const status = await this.storage.getStatus();
-    if (status.snapshot_done && status.checkpoint_lsn) {
+    if (status.snapshotDone) {
       this.logger.info(`Initial replication already done`);
       return { needsInitialSync: false, snapshotLsn: null };
     }
 
-    return { needsInitialSync: true, snapshotLsn: status.snapshot_lsn };
+    return { needsInitialSync: true, snapshotLsn: status.resumeLsn };
   }
 
   async setupCheckpointsCollection() {
@@ -277,20 +275,16 @@ export class MongoSnapshotter {
     }
 
     const status = await this.storage.getStatus();
-    if (status.snapshot_done) {
+    if (status.snapshotDone) {
       return;
     }
 
-    const lastOp = this.lastSnapshotOpId ?? status.keepalive_op;
-    if (lastOp != null) {
-      // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream.
-      // Keeping snapshot_done false until this completes makes this resumable after interruption.
-      await this.storage.populatePersistentChecksumCache({
-        // No checkpoint yet, but we do have the opId.
-        maxOpId: lastOp,
-        signal: this.abortSignal
-      });
-    }
+    // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream.
+    // Keeping snapshot_done false until this completes makes this resumable after interruption.
+    // No checkpoint exists yet - storage defaults to its highest persisted op id.
+    await this.storage.populatePersistentChecksumCache({
+      signal: this.abortSignal
+    });
 
     if (this.queue.size != 0) {
       return;
@@ -345,13 +339,10 @@ export class MongoSnapshotter {
     const noCheckpointBefore = await createCheckpoint(this.client, this.defaultDb, STANDALONE_CHECKPOINT_ID);
     await writer.markTableSnapshotDone([table], noCheckpointBefore);
 
-    // This commit ensures we set keepalive_op.
+    // This commit durably records the persisted ops, so a later checkpoint covers them.
     const resumeLsn = writer.resumeFromLsn ?? MongoLSN.ZERO.comparable;
     await writer.commit(resumeLsn);
 
-    if (writer.last_flushed_op != null) {
-      this.lastSnapshotOpId = writer.last_flushed_op;
-    }
     this.logger.info(`Flushed snapshot at ${writer.last_flushed_op}`);
   }
 
@@ -445,10 +436,7 @@ export class MongoSnapshotter {
       }
 
       // Important: flush before marking progress
-      const result = await batch.flush();
-      if (result?.flushed_op != null) {
-        this.lastSnapshotOpId = result.flushed_op;
-      }
+      await batch.flush();
       at += docBatch.length;
       rowsReplicatedMetric.add(docBatch.length);
 
@@ -481,10 +469,12 @@ export class MongoSnapshotter {
       // Ignore the postImages check in this case.
     }
 
+    // Note: resolveTables uses the batch's own parsed sync config set. Passing
+    // this.syncRules here would pair sources from one parse with the batch's mapping
+    // from another parse.
     const result = await batch.resolveTables({
       connection_id: this.connectionId,
-      source: descriptor,
-      syncRules: this.syncRules
+      source: descriptor
     });
 
     // Drop conflicting collections.

package/test/src/change_stream.test.ts

@@ -820,9 +820,9 @@ bucket_definitions:
 
     // Simulate an error
     await context.storage!.reportError(new Error('simulated error'));
-    const syncRules = await context.factory.getActiveSyncRulesContent();
+    const syncRules = (await context.factory.getActiveSyncConfig())?.content;
     expect(syncRules).toBeTruthy();
-    expect(syncRules?.last_fatal_error).toEqual('simulated error');
+    expect((await syncRules?.getSyncConfigStatus())?.last_fatal_error).toEqual('simulated error');
 
     // The next checkpoint should clear the error.
     await context.getCheckpoint();
@@ -830,7 +830,8 @@ bucket_definitions:
     // Just wait, and check that the error is cleared automatically.
     await vi.waitUntil(
       async () => {
-        const error = (await context.factory.getActiveSyncRulesContent())?.last_fatal_error;
+        const syncRules = (await context.factory.getActiveSyncConfig())?.content;
+        const error = (await syncRules?.getSyncConfigStatus())?.last_fatal_error;
         return error == null;
       },
       { timeout: 2_000 }

package/test/src/change_stream_utils.ts

@@ -30,7 +30,7 @@ export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
   private abortController = new AbortController();
   private settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
-  private syncRulesContent?: storage.PersistedSyncRulesContent;
+  private syncRulesContent?: storage.PersistedSyncConfigContent;
   public storage?: SyncRulesBucketStorage;
 
   /**
@@ -101,26 +101,26 @@ export class ChangeStreamTestContext {
   }
 
   async updateSyncRules(content: string) {
-    const syncRules = await this.factory.updateSyncRules(
+    const replicationStream = await this.factory.updateSyncRules(
       updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
     );
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = replicationStream.syncConfigContent[0];
+    this.storage = this.factory.getInstance(replicationStream);
     return this.storage!;
   }
 
   async loadNextSyncRules() {
-    const syncRules = await this.factory.getNextSyncRulesContent();
-    if (syncRules == null) {
+    const syncConfig = await this.factory.getDeployingSyncConfig();
+    if (syncConfig == null) {
       throw new Error(`Next sync config not available`);
     }
 
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = syncConfig.content;
+    this.storage = syncConfig.storage;
     return this.storage!;
   }
 
-  private getSyncRulesContent(): storage.PersistedSyncRulesContent {
+  private getSyncConfigContent(): storage.PersistedSyncConfigContent {
     if (this.syncRulesContent == null) {
       throw new Error('Sync config not configured - call updateSyncRules() first');
     }
@@ -197,8 +197,8 @@ export class ChangeStreamTestContext {
 
   async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
-    const syncRules = this.getSyncRulesContent();
-    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start));
+    const syncConfigContent = this.getSyncConfigContent();
+    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncConfigContent, bucket, start));
     return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
@@ -207,9 +207,9 @@ export class ChangeStreamTestContext {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
-    const syncRules = this.getSyncRulesContent();
+    const syncConfigContent = this.getSyncConfigContent();
     const checkpoint = await this.getCheckpoint(options);
-    let map = [bucketRequest(syncRules, bucket, start)];
+    let map = [bucketRequest(syncConfigContent, bucket, start)];
     let data: OplogEntry[] = [];
     while (true) {
       const batch = this.storage!.getBucketDataBatch(checkpoint, map);
@@ -219,15 +219,15 @@ export class ChangeStreamTestContext {
       if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
         break;
       }
-      map = [bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))];
+      map = [bucketRequest(syncConfigContent, bucket, BigInt(batches[0]!.chunkData.next_after))];
     }
     return data;
   }
 
   async getChecksums(buckets: string[], options?: { timeout?: number }): Promise<utils.ChecksumMap> {
     let checkpoint = await this.getCheckpoint(options);
-    const syncRules = this.getSyncRulesContent();
-    const versionedBuckets = buckets.map((bucket) => bucketRequest(syncRules, bucket, 0n));
+    const syncConfigContent = this.getSyncConfigContent();
+    const versionedBuckets = buckets.map((bucket) => bucketRequest(syncConfigContent, bucket, 0n));
     const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
 
     const unversioned: utils.ChecksumMap = new Map();
@@ -258,7 +258,7 @@ export async function getClientCheckpoint(
   let lastCp: ReplicationCheckpoint | null = null;
 
   while (Date.now() - start < timeout) {
-    const storage = await storageFactory.getActiveStorage();
+    const storage = (await storageFactory.getActiveSyncConfig())?.storage;
     const cp = await storage?.getCheckpoint();
     if (cp != null) {
       lastCp = cp;

package/test/src/checkpoint_retry.test.ts

@@ -20,7 +20,7 @@ describe('checkpoint retryable writes', () => {
     // It is quite difficult to simulate this, even with failCommand. We currently rely on triggering a socket timeout for the first command,
     // with another write happening right after that.
 
-    const TIMEOUT = 100;
+    const TIMEOUT = 200;
     const INSERT_COUNT = 5;
 
     const { db, client } = await connectMongoData({

package/test/src/resume.test.ts

@@ -67,8 +67,8 @@ function defineResumeTest({ factory: factoryGenerator, storageVersion }: Storage
 
     // Create a new context without updating the sync config
     await using context2 = new ChangeStreamTestContext(factory, connectionManager, {}, storageVersion);
-    const activeContent = await factory.getActiveSyncRulesContent();
-    context2.storage = factory.getInstance(activeContent!);
+    const active = await factory.getActiveSyncConfig();
+    context2.storage = active!.storage;
 
     // If this test times out, it likely didn't throw the expected error here.
     const result = await context2.startStreaming();

package/test/src/util.ts

@@ -21,7 +21,8 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
 
 export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({
   url: env.MONGO_TEST_URL,
-  isCI: env.CI
+  isCI: env.CI,
+  supportsMultipleSyncConfigs: true
 });
 
 export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestSetup({

package/tsconfig.json

@@ -2,10 +2,7 @@
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
     "rootDir": "src",
-    "outDir": "dist",
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "sourceMap": true
+    "outDir": "dist"
   },
   "include": ["src"],
   "references": [

package/tsconfig.scripts.json

@@ -6,8 +6,7 @@
     "target": "esnext",
     "noEmit": true,
     "composite": false,
-    "types": ["node"],
-    "skipLibCheck": true
+    "types": ["node"]
   },
   "include": ["scripts/**/*.mts"]
 }