@powersync/service-module-mongodb 0.21.2 → 0.22.0

package/dist/test-utils/test-utils-index.d.ts

@@ -0,0 +1 @@
+export * from './ChangeStreamTestContext.js';

package/dist/test-utils/test-utils-index.js

@@ -0,0 +1,2 @@
+export * from './ChangeStreamTestContext.js';
+//# sourceMappingURL=test-utils-index.js.map

package/dist/test-utils/test-utils-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-utils-index.js","sourceRoot":"","sources":["../../src/test-utils/test-utils-index.ts"],"names":[],"mappings":"AAAA,cAAc,8BAA8B,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@powersync/service-module-mongodb",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "version": "0.21.2",
+  "version": "0.22.0",
   "license": "FSL-1.1-ALv2",
   "type": "module",
   "publishConfig": {
@@ -15,22 +15,23 @@
     "bson": "^6.10.4",
     "ts-codec": "^1.3.0",
     "uuid": "^14.0.0",
-    "@powersync/lib-service-mongodb": "0.6.33",
-    "@powersync/lib-services-framework": "0.10.3",
-    "@powersync/service-core": "1.27.0",
+    "@powersync/lib-service-mongodb": "0.6.34",
+    "@powersync/lib-services-framework": "0.10.4",
+    "@powersync/service-core": "1.27.1",
     "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-sync-rules": "0.43.0",
-    "@powersync/service-types": "0.19.0"
+    "@powersync/service-sync-rules": "0.44.0",
+    "@powersync/service-types": "0.20.0"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.18.3",
-    "@powersync/service-module-mongodb-storage": "0.23.0",
-    "@powersync/service-module-postgres-storage": "0.19.2"
+    "@powersync/service-core-tests": "0.18.4",
+    "@powersync/service-module-mongodb-storage": "0.23.1",
+    "@powersync/service-module-postgres-storage": "0.19.3"
   },
   "scripts": {
     "build": "tsc -b",
     "build:tests": "tsc -b test/tsconfig.json",
     "clean": "rm -rf ./dist && tsc -b --clean",
-    "test": "vitest"
+    "test": "vitest",
+    "benchmark:replication": "vitest run --config vitest.benchmark.config.ts"
   }
 }

package/src/api/MongoRouteAPIAdapter.ts

@@ -4,7 +4,7 @@ import { api, ParseSyncConfigOptions, ReplicationHeadCallback } from '@powersync
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as service_types from '@powersync/service-types';
 
-import { logger } from '@powersync/lib-services-framework';
+import { ErrorCode, logger } from '@powersync/lib-services-framework';
 import { CheckpointImplementation } from '../replication/checkpoints/CheckpointImplementation.js';
 import { createCheckpointImplementation } from '../replication/checkpoints/create-checkpoint-implementation.js';
 import { MongoManager } from '../replication/MongoManager.js';
@@ -35,10 +35,34 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
 
   getParseSyncRulesOptions(): ParseSyncConfigOptions {
     return {
-      defaultSchema: this.defaultSchema
+      defaultSchema: this.defaultSchema,
+      connections: { [this.connectionTag]: { type: this.config.type } }
     };
   }
 
+  async validateSourceCapabilities(config: sync_rules.SyncConfig): Promise<service_types.ValidationDiagnostic[]> {
+    return this.validateMongoFilters(config);
+  }
+
+  /**
+   * Validate MongoDB pre-filtering support. External modules can override this method to account for
+   * their plan parameters and filter capabilities while retaining other source-capability checks.
+   */
+  protected async validateMongoFilters(config: sync_rules.SyncConfig): Promise<service_types.ValidationDiagnostic[]> {
+    const filtered = sync_rules.hasMongoFilterExpressions(
+      config instanceof sync_rules.PrecompiledSyncConfig ? (config.plan.sourceTableConfig ?? {}) : {},
+      this.connectionTag
+    );
+    return filtered
+      ? [
+          {
+            level: 'fatal',
+            message: `[${ErrorCode.PSYNC_S1348}] MongoDB replication pre-filtering is not supported in this edition of the PowerSync service.`
+          }
+        ]
+      : [];
+  }
+
   async shutdown(): Promise<void> {
     await this.client.close();
   }

package/src/index.ts

@@ -1,4 +1,5 @@
 export * from './api/MongoRouteAPIAdapter.js';
 export * from './module/MongoModule.js';
 export * from './replication/replication-index.js';
+export * as test_utils from './test-utils/test-utils-index.js';
 export * from './types/types.js';

package/src/module/MongoModule.ts

@@ -37,6 +37,7 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
       id: this.getDefaultId(normalisedConfig.database ?? ''),
       syncRuleProvider: syncRuleProvider,
       storageEngine: context.storageEngine,
+      configuration: context.configuration,
       metricsEngine: context.metricsEngine,
       connectionFactory: connectionFactory,
       rateLimiter: new MongoErrorRateLimiter(),
@@ -47,7 +48,7 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
   /**
    * Combines base config with normalized connection settings
    */
-  private resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig {
+  protected resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig {
     return {
       ...config,
       ...types.normalizeConnectionConfig(config)

package/src/replication/ChangeStream.ts

@@ -18,7 +18,7 @@ import {
   SourceTable,
   storage
 } from '@powersync/service-core';
-import { HydratedSyncConfig } from '@powersync/service-sync-rules';
+import { hasMongoFilterExpressions, HydratedSyncConfig } from '@powersync/service-sync-rules';
 import { ReplicationMetric } from '@powersync/service-types';
 import { performance } from 'node:perf_hooks';
 import { PostImagesOption } from '../types/types.js';
@@ -27,18 +27,20 @@ import { CheckpointImplementation } from './checkpoints/CheckpointImplementation
 import { createCheckpointImplementation } from './checkpoints/create-checkpoint-implementation.js';
 import { MongoManager } from './MongoManager.js';
 import { getCacheIdentifier, getMongoRelation } from './MongoRelation.js';
-import { MongoSnapshotter, MongoSnapshotterHooks } from './MongoSnapshotter.js';
 import {
-  ChangeStreamBatch,
-  parseChangeDocument,
-  ProjectedChangeStreamDocument,
-  rawChangeStream
-} from './RawChangeStream.js';
+  DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
+  MongoReplicationQueryProvider,
+  MongoReplicationQueryProviderFactory
+} from './MongoReplicationQueryProvider.js';
+import { openMongoReplicationStream } from './MongoReplicationStream.js';
+import type { MongoReplicationStreamBatch } from './MongoReplicationStreamTypes.js';
+import { MongoSnapshotter, MongoSnapshotterHooks } from './MongoSnapshotter.js';
+import { ChangeStreamBatch, ProjectedChangeStreamDocument } from './RawChangeStream.js';
 import { CHECKPOINTS_COLLECTION, detectDocumentDb, timestampToDate } from './replication-utils.js';
 import { DirectSourceRowConverter, SourceRowConverter } from './SourceRowConverter.js';
 export interface ChangeStreamOptions {
-  connections: MongoManager;
   storage: storage.SyncRulesBucketStorage;
+  connections: MongoManager;
   metrics: MetricsEngine;
   abort_signal: AbortSignal;
   /**
@@ -61,6 +63,10 @@ export interface ChangeStreamOptions {
 
   storageHooks?: storage.StorageHooks;
   snapshotHooks?: MongoSnapshotterHooks;
+  /**
+   * Create one adapter per replication attempt, shared with its snapshotter and source probes.
+   */
+  createReplicationQueryProvider?: MongoReplicationQueryProviderFactory;
 
   logger?: Logger;
 }
@@ -118,6 +124,8 @@ export class ChangeStream {
   private storageHooks: storage.StorageHooks | undefined;
 
   private readonly sourceRowConverter: SourceRowConverter;
+  private readonly queryProvider: MongoReplicationQueryProvider;
+  private readonly hasQueryProviderFactory: boolean;
 
   private keepaliveIntervalMs: number;
 
@@ -130,7 +138,7 @@ export class ChangeStream {
   private lastPersistedResumeTimestamp = 0;
   private lastBatchCheckpoint = 0;
 
-  constructor(options: ChangeStreamOptions) {
+  constructor(private readonly options: ChangeStreamOptions) {
     this.storage = options.storage;
     this.metrics = options.metrics;
     this.group_id = options.storage.replicationStreamId;
@@ -150,6 +158,13 @@ export class ChangeStream {
     this.sync_rules = options.storage.getParsedSyncRules({
       defaultSchema: this.defaultDb.databaseName
     });
+    this.hasQueryProviderFactory = options.createReplicationQueryProvider != null;
+    this.queryProvider =
+      options.createReplicationQueryProvider?.({
+        syncConfig: this.sync_rules,
+        connectionTag: this.connections.connectionTag,
+        defaultSchema: this.defaultDb.databaseName
+      }) ?? DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER;
     this.sourceRowConverter = new DirectSourceRowConverter(this.sync_rules.compatibility);
 
     // The change stream aggregation command should timeout before the socket times out,
@@ -166,7 +181,8 @@ export class ChangeStream {
       ...options,
       abortSignal: this.abortSignal,
       logger: snapshotLogger,
-      checkpointStreamId: this.checkpointStreamId
+      checkpointStreamId: this.checkpointStreamId,
+      queryProvider: this.queryProvider
     });
 
     options.abort_signal.addEventListener(
@@ -193,7 +209,9 @@ export class ChangeStream {
     return this.connections.options.postImages == PostImagesOption.AUTO_CONFIGURE;
   }
 
-  /** The active checkpoint strategy. Only valid after ensureDetected(). */
+  /**
+   * The active checkpoint strategy. Only valid after ensureDetected().
+   */
   private get checkpointImplementation(): CheckpointImplementation {
     if (this._checkpointImplementation == null) {
       throw new ReplicationAssertionError('Checkpoint implementation not initialized - call ensureDetected() first');
@@ -216,6 +234,7 @@ export class ChangeStream {
         'Azure DocumentDB support is in alpha. APIs and behavior may change, and long-term stability is not yet guaranteed.'
       );
     }
+    await this.queryProvider.validateSource?.({ connectionManager: this.connections, isDocumentDb: this.isDocumentDb });
     this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
       client: this.client,
       db: this.defaultDb,
@@ -485,6 +504,17 @@ export class ChangeStream {
   }
 
   private async initReplication() {
+    // A previously deployed config may outlive the module that provided pre-filtering.
+    // Throw inside replicate() so the failure is persisted in replication diagnostics.
+    if (
+      !this.hasQueryProviderFactory &&
+      hasMongoFilterExpressions(this.sync_rules.sourceTableConfig, this.connections.connectionTag)
+    ) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1348,
+        'This sync config requires MongoDB pre-filtering, but no query provider factory is registered. Restore the required module, or remove the filters and deploy a new sync config to restart replication.'
+      );
+    }
     const result = await this.snapshotter.checkSlot();
     await this.snapshotter.setupCheckpointsCollection();
     if (result.needsInitialSync) {
@@ -512,7 +542,7 @@ export class ChangeStream {
       if (
         e instanceof mongo.MongoServerError &&
         e.codeName == 'NoMatchingDocument' &&
-        e.errmsg?.includes('post-image was not found')
+        (e.errmsg?.includes('post-image was not found') || e.errmsg?.includes('pre-image was not found'))
       ) {
         throw new ChangeStreamInvalidatedError(e.errmsg, e);
       }
@@ -520,86 +550,32 @@ export class ChangeStream {
     }
   }
 
-  private rawChangeStreamBatches(options: {
+  private openChangeStream(options: {
     lsn: string | null;
     maxAwaitTimeMS?: number;
     batchSize?: number;
     filters: { $match: any; multipleDatabases: boolean };
     signal?: AbortSignal;
     tracer?: PerformanceTracer<'changestream'>;
-  }): AsyncIterableIterator<ChangeStreamBatch> {
-    const position = options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null;
-    const startAfter = position?.startAfter ?? undefined;
-    const resumeAfter = position?.resumeAfter ?? undefined;
-
-    const filters = options.filters;
-
-    let fullDocument: 'required' | 'updateLookup';
-
-    if (this.isDocumentDb) {
-      // DocumentDB does not support changeStreamPreAndPostImages, so 'required' won't work.
-      fullDocument = 'updateLookup';
-    } else if (this.usePostImages) {
-      // 'read_only' or 'auto_configure'
-      // Configuration happens during snapshot, or when we see new
-      // collections.
-      fullDocument = 'required';
-    } else {
-      fullDocument = 'updateLookup';
-    }
-    const streamOptions: mongo.ChangeStreamOptions & mongo.Document = {
-      fullDocument: fullDocument
-    };
-    if (!this.isDocumentDb) {
-      // DocumentDB does not support showExpandedEvents.
-      streamOptions.showExpandedEvents = true;
-    }
-    const pipeline: mongo.Document[] = [
-      {
-        $changeStream: streamOptions
-      },
-      {
-        $match: filters.$match
+    onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
+  }): AsyncIterableIterator<MongoReplicationStreamBatch> {
+    return openMongoReplicationStream({
+      db: this.defaultDb,
+      queryProvider: this.queryProvider,
+      namespaceFilter: options.filters,
+      isDocumentDb: this.isDocumentDb,
+      usePostImages: this.usePostImages,
+      position: options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null,
+      skipInitialTimestamp: true,
+      onBatch: options.onBatch,
+      options: {
+        batchSize: options.batchSize ?? this.snapshotChunkLength,
+        maxAwaitTimeMS: options.maxAwaitTimeMS ?? this.maxAwaitTimeMS,
+        maxTimeMS: this.changeStreamTimeout,
+        signal: options.signal,
+        logger: this.logger,
+        tracer: options.tracer
       }
-    ];
-    if (!this.isDocumentDb) {
-      // DocumentDB does not support $changeStreamSplitLargeEvent.
-      pipeline.push({ $changeStreamSplitLargeEvent: {} });
-    }
-
-    /**
-     * Only one of these options can be supplied at a time.
-     */
-    if (resumeAfter) {
-      streamOptions.resumeAfter = resumeAfter;
-    } else if (startAfter != null) {
-      // Legacy: We don't persist lsns without resumeTokens anymore, but we do still handle the
-      // case if we have an old one.
-      // This is also relevant for getSnapshotLSN().
-      // The sentinel implementation never produces a startAfter, and a fresh DocumentDB stream
-      // opens from "now" with neither option set.
-      streamOptions.startAtOperationTime = startAfter;
-    }
-
-    let watchDb: mongo.Db;
-    if (this.isDocumentDb || filters.multipleDatabases) {
-      // DocumentDB only supports cluster-level change streams.
-      watchDb = this.client.db('admin');
-      streamOptions.allChangesForCluster = true;
-    } else {
-      watchDb = this.defaultDb;
-    }
-
-    const maxAwaitTimeMS = options.maxAwaitTimeMS ?? this.maxAwaitTimeMS;
-
-    return rawChangeStream(watchDb, pipeline, {
-      batchSize: options.batchSize ?? this.snapshotChunkLength,
-      maxAwaitTimeMS,
-      maxTimeMS: this.changeStreamTimeout,
-
-      signal: options.signal,
-      logger: this.logger,
-      tracer: options.tracer
     });
   }
 
@@ -638,22 +614,31 @@ export class ChangeStream {
         if (resumeFromLsn == null) {
           throw new ReplicationAssertionError(`No LSN found to resume from`);
         }
-        // Seed the implementation's coordinate state from the stored LSN, and parse
-        // the legacy startAfter timestamp (timestamp implementation only) for the
-        // resume-boundary dedupe guard below.
+        // Seed the checkpoint strategy's coordinate from the durable source position.
         this.checkpointImplementation.seedPosition(resumeFromLsn);
-        const { startAfter } = this.checkpointImplementation.parseResumePosition(resumeFromLsn);
         let outerSpan = tracer.span('batch');
 
         this.checkpointImplementation.logResume(resumeFromLsn);
 
         const filters = this.getSourceNamespaceFilters();
         // This is closed when the for loop below returns/breaks/throws
-        const batchStream = this.rawChangeStreamBatches({
+        let processingSpan: ReturnType<typeof tracer.span> | undefined;
+        let receivedBytes = 0;
+        let changesSinceProgress = 0;
+        let filteredSinceProgress: number | null = null;
+        const stream = this.openChangeStream({
           lsn: resumeFromLsn,
           filters,
           signal: this.abortSignal,
-          tracer
+          tracer,
+          onBatch: (batch) => {
+            // Count actual transport, even if the adapter drops every envelope or reassembles fragments.
+            bytesReplicatedMetric.add(batch.byteSize);
+            chunksReplicatedMetric.add(1);
+            receivedBytes += batch.byteSize;
+            processingSpan = tracer.span('processing');
+            return processingSpan;
+          }
         });
 
         // Always start with a checkpoint.
@@ -661,113 +646,15 @@ export class ChangeStream {
         // no data to replicate.
         let waitForCheckpointLsn: string | null = await this.createBatchCheckpoint();
 
-        let splitDocument: ProjectedChangeStreamDocument | null = null;
-
-        let flexDbNameWorkaroundLogged = false;
-
-        let lastEmptyResume = performance.now();
+        let lastKeepalive = performance.now();
         let lastTxnKey: string | null = null;
 
-        for await (let eventBatch of batchStream) {
-          const { events, resumeToken } = eventBatch;
-          using batchSpan = tracer.span('processing');
-
-          bytesReplicatedMetric.add(eventBatch.byteSize);
-          chunksReplicatedMetric.add(1);
-          if (this.abortSignal.aborted) {
-            break;
-          }
-          this.touch();
-          if (events.length == 0) {
-            // No changes in this batch, but we still want to persist progress.
-            // We do this by persisting a keepalive checkpoint.
-            // If we don't update it on empty events, we do keep consistency, but resuming the stream
-            // with old tokens may cause connection timeouts.
-            const hadRecentKeepalive = performance.now() - lastEmptyResume < this.keepaliveIntervalMs;
-            if (waitForCheckpointLsn == null && !hadRecentKeepalive) {
-              // Case 1: We have no changes, and we are not waiting for a checkpoint to be created,
-              // and we have not recently persisted a keepalive. Persist one now, and call setResumeLsn() below.
-              // This is the normal case for an idle stream.
-              // The implementation persists a keepalive (timestamp) or bumps the
-              // sentinel so a later event commits (sentinel). Logging is handled
-              // inside the implementation.
-              await this.checkpointImplementation.keepalive(batch, resumeToken);
-              this.touch();
-              lastEmptyResume = performance.now();
-              this.replicationLag.markStarted();
-            } else if (hadRecentKeepalive) {
-              // Case 2: We have no changes, and may or may not be waiting for a checkpoint to be created.
-              // We have recently persisted a keepalive.
-              // Continue waiting.
-              continue;
-            } else {
-              // Case 3: Waiting for a checkpoint; have not had a recent keepalive.
-              // We cannot call checkpointImplementation.keepalive() here, but we do call
-              // setResumeLsn() below.
-            }
-          }
-
-          this.touch();
-
-          for (let eventIndex = 0; eventIndex < events.length; eventIndex++) {
-            const rawChangeDocument = events[eventIndex];
-            const originalChangeDocument = parseChangeDocument(rawChangeDocument);
-            if (this.abortSignal.aborted) {
-              break;
-            }
-
-            if (startAfter != null && originalChangeDocument.clusterTime?.lte(startAfter)) {
-              continue;
-            }
-
-            let changeDocument = originalChangeDocument;
-            if (originalChangeDocument?.splitEvent != null) {
-              // Handle split events from $changeStreamSplitLargeEvent.
-              // This is only relevant for very large update operations.
-              const splitEvent = originalChangeDocument?.splitEvent;
-
-              if (splitDocument == null) {
-                splitDocument = originalChangeDocument;
-              } else {
-                splitDocument = Object.assign(splitDocument, originalChangeDocument);
-              }
-
-              if (splitEvent.fragment == splitEvent.of) {
-                // Got all fragments
-                changeDocument = splitDocument;
-                splitDocument = null;
-              } else {
-                // Wait for more fragments
-                continue;
-              }
-            } else if (splitDocument != null) {
-              // We were waiting for fragments, but got a different event
-              throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
-            }
-
-            if (
-              !filters.multipleDatabases &&
-              'ns' in changeDocument &&
-              changeDocument.ns.db != this.defaultDb.databaseName &&
-              changeDocument.ns.db.endsWith(`_${this.defaultDb.databaseName}`)
-            ) {
-              // When all of the following conditions are met:
-              // 1. We're replicating from an Atlas Flex instance.
-              // 2. There were changestream events recorded while the PowerSync service is paused.
-              // 3. We're only replicating from a single database.
-              // Then we've observed an ns with for example {db: '67b83e86cd20730f1e766dde_ps'},
-              // instead of the expected {db: 'ps'}.
-              // We correct this.
-              changeDocument.ns.db = this.defaultDb.databaseName;
-
-              if (!flexDbNameWorkaroundLogged) {
-                flexDbNameWorkaroundLogged = true;
-                this.logger.warn(
-                  `Incorrect DB name in change stream: ${changeDocument.ns.db}. Changed to ${this.defaultDb.databaseName}.`
-                );
-              }
-            }
-
+        streamLoop: for await (const sourceBatch of stream) {
+          if (this.abortSignal.aborted) break;
+          for (const { event: changeDocument, hasBufferedChanges } of sourceBatch.events) {
+            if (this.abortSignal.aborted) break streamLoop;
+            this.touch();
+            changesSinceProgress++;
             const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
 
             if (ns?.coll == CHECKPOINTS_COLLECTION) {
@@ -821,7 +708,6 @@ export class ChangeStream {
                 // Once there is already a batch checkpoint pending, or the driver has buffered more
                 // change stream events, collapse standalone checkpoints into the normal batch
                 // checkpoint flow to avoid commit churn under sustained load.
-                const hasBufferedChanges = eventIndex < events.length - 1;
                 if (hasBufferedChanges && waitForCheckpointLsn == null) {
                   // Buffered changes - create a new batch checkpoint to rate limit commits
                   using _ = tracer.span('source_checkpoint');
@@ -926,40 +812,71 @@ export class ChangeStream {
               });
             }
           }
+          if (this.abortSignal.aborted) break;
+          this.touch();
+          const batchFilteredCount = sourceBatch.metadata.filteredCount;
+          if (batchFilteredCount != null) {
+            filteredSinceProgress = (filteredSinceProgress ?? 0) + batchFilteredCount;
+          }
+          const resumeToken = sourceBatch.getResumeToken();
+          if (resumeToken != null) {
+            const filteredCount = filteredSinceProgress;
+            if (changesSinceProgress == 0) {
+              // No retained changes since the previous progress boundary: this is either idle or filtered-only traffic.
+              // Case 1: No pending barrier and the keepalive interval has elapsed. Advance checkpoints as below,
+              // then flush and save the resume token.
+              // Case 2: Idle traffic with a recent keepalive. Skip this resume update, whether or not a barrier
+              // is pending, to preserve the idle throttle.
+              // Case 3: All other combinations. Flush and save the resume token without requesting a checkpoint.
+              // A pending barrier will provide the checkpoint boundary once replication reaches it. Filtered
+              // progress still saves its token even during the keepalive interval, so excluded work is resumable.
+              const hadRecentKeepalive = performance.now() - lastKeepalive < this.keepaliveIntervalMs;
+              if (waitForCheckpointLsn == null && !hadRecentKeepalive) {
+                if ((filteredCount ?? 0) > 0) {
+                  // Case 1a: A transaction can span batches: this batch may contain only filtered changes,
+                  // while retained changes from the same transaction are still unread. They share a timestamp,
+                  // so publishing a checkpoint here could acknowledge the transaction before its data is saved.
+                  // Request a source barrier; reaching it ensures those retained changes have been processed.
+                  // The progress token can still be saved below for recovery without publishing a checkpoint.
+                  using _ = tracer.span('source_checkpoint');
+                  waitForCheckpointLsn = await this.createBatchCheckpoint();
+                } else {
+                  // Case 1b: Idle traffic. The timestamp implementation persists a keepalive directly;
+                  // the sentinel implementation writes a source marker whose event will commit later.
+                  await this.checkpointImplementation.keepalive(batch, resumeToken);
+                  this.replicationLag.markStarted();
+                }
+                this.touch();
+                lastKeepalive = performance.now();
+              } else if ((filteredCount ?? 0) == 0 && hadRecentKeepalive) {
+                // Case 2: Only idle progress skips resume persistence. Cases 1 and 3 fall through below.
+                continue;
+              }
+            }
 
-          if (splitDocument == null) {
-            // We flush and mark progress on every batch of data we receive.
-            // Batches are generally large (64MB or 6000 events, whichever comes first),
-            // so this is a good natural point to flush and mark progress.
-            // We avoid this when splitDocument is set, since we cannot resume in the middle of a split event.
             const { lsn, timestamp } = this.checkpointImplementation.lsnFromResumeToken(resumeToken);
+            // Row writes must be durable before their source token. This advances recovery without
+            // publishing a checkpoint ahead of an outstanding barrier, transaction or snapshot.
             await batch.flush({ oldestUncommittedChange: this.replicationLag.oldestUncommittedChange });
-            // TODO: We should consider making this standard behavior of flush().
             await batch.setResumeLsn(lsn);
-
-            if (timestamp != null) {
-              // Note that this timestamp provided by MongoDB is not exact - it can be around 10s behind.
-              this.lastPersistedResumeTimestamp = timestamp.getTime();
-            } else {
-              // DocumentDB: No timestamp associated with the resumeToken. Just use the current time.
-              this.lastPersistedResumeTimestamp = Date.now();
-            }
-          }
-
-          batchSpan.end();
-          const durationsMicroseconds = outerSpan.end();
-          const duration = batchSpan.durationMillis;
-
-          this.logger.info(
-            `Processed batch of ${events.length} changes / ${eventBatch.byteSize} bytes in ${duration}ms`,
-            {
-              count: events.length,
-              bytes: eventBatch.byteSize,
-              duration,
+            // MongoDB's token timestamp may lag by about 10 seconds. DocumentDB tokens have no timestamp.
+            this.lastPersistedResumeTimestamp = timestamp?.getTime() ?? Date.now();
+            processingSpan?.end();
+            const durationsMicroseconds = outerSpan.end();
+            const filteringDescription = filteredCount == null ? '' : ` and ${filteredCount} filtered events`;
+            this.logger.info(`Processed ${changesSinceProgress} changes${filteringDescription}`, {
+              count: changesSinceProgress,
+              ...(filteredCount == null ? {} : { filteredCount }),
+              bytes: receivedBytes,
+              duration: processingSpan?.durationMillis,
               t: durationsMicroseconds
-            }
-          );
-          outerSpan = tracer.span('batch');
+            });
+            changesSinceProgress = 0;
+            filteredSinceProgress = null;
+            receivedBytes = 0;
+            outerSpan = tracer.span('batch');
+            continue;
+          }
         }
       }
     );