@powersync/service-module-mongodb 0.0.0-dev-20261007110800 → 0.0.0-dev-20261015071128

package/src/replication/ChangeStreamReplicationJob.ts

@@ -70,15 +70,22 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
       if (this.isStopped) {
         return;
       }
-      const stream = new ChangeStream({
-        abort_signal: this.abortController.signal,
-        storage: this.options.storage,
-        metrics: this.options.metrics,
-        connections: connectionManager,
-        keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
-        logger: this.logger,
-        createReplicationQueryProvider: this.createReplicationQueryProvider
-      });
+      let stream: ChangeStream;
+      try {
+        stream = new ChangeStream({
+          abort_signal: this.abortController.signal,
+          storage: this.options.storage,
+          metrics: this.options.metrics,
+          connections: connectionManager,
+          keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
+          logger: this.logger,
+          createReplicationQueryProvider: this.createReplicationQueryProvider
+        });
+      } catch (error) {
+        // Construction can fail before ChangeStream.replicate() reports errors.
+        await this.storage.reportError(error);
+        throw error;
+      }
       this.lastStream = stream;
       await stream.replicate();
     } finally {

package/src/replication/MongoReplicationQueryProvider.ts

@@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { SourceTable } from '@powersync/service-core';
 import { HydratedSyncConfig } from '@powersync/service-sync-rules';
 import { MongoManager } from './MongoManager.js';
-import { MongoReplicationStreamContext, MongoReplicationStreamItem } from './MongoReplicationStream.js';
+import type { MongoReplicationStreamBatch, MongoReplicationStreamContext } from './MongoReplicationStreamTypes.js';
 
 /**
  * Sync config and source namespace information used to construct a provider for a replication attempt.
@@ -35,17 +35,15 @@ export interface MongoReplicationQueryProvider {
   validateSource?(options: { connectionManager: MongoManager; isDocumentDb: boolean }): Promise<void>;
 
   /**
-   * Call context.open() with any additional stages/pre-image options, then optionally wrap its iterator.
-   * Unlike pipeline stages alone, the wrapper can exclude marked events while still forwarding progress.
-   * Yield changes and safe progress, including when all data events are excluded.
-   * Count excluded complete events in the next progress item's filteredCount, resetting it after each
-   * progress item. Preserve the reader's safe resume token rather than advancing past buffered changes.
-   * An exit from a filtered set may be represented as a synthetic delete retaining its original key and token.
-   * Checkpoint/DDL events must remain ordered and intact. Progress alone must never stand in for a checkpoint.
-   * Each call opens a separate iterator, also used for snapshot barriers and resume validation. Keep
-   * per-iterator state separate and close the underlying iterator when the wrapper exits.
+   * Call context.open() with any additional stages/pre-image options. For filtering, pass its result
+   * to filterMongoReplicationStream({ batches, filter }) with a MongoReplicationFilter callback.
+   * The helper preserves checkpoints, DDL, batch boundaries, buffered-change flags and safe progress,
+   * counts excluded events, and closes the source iterator on failure or early exit.
+   * The default provider returns the opener's batches directly.
+   * Each invocation is independent and also serves snapshot barriers and resume validation; keep
+   * filter state local to that invocation. Snapshot selection must agree with streaming decisions.
    */
-  openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<MongoReplicationStreamItem>;
+  openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<MongoReplicationStreamBatch>;
 
   /**
    * Returns an additional MongoDB find predicate for a physical source table.

package/src/replication/MongoReplicationStream.ts

@@ -1,6 +1,16 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { Logger, ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { buildMongoChangeStreamPipeline, MongoReplicationQueryProvider } from './MongoReplicationQueryProvider.js';
+import {
+  MongoReplicationFilterOptions,
+  MongoReplicationFilterResult,
+  MongoReplicationParserOptions,
+  MongoReplicationStreamBatch,
+  MongoReplicationStreamBatchMetadata,
+  MongoReplicationStreamChange,
+  MongoReplicationStreamOptions,
+  RawMongoReplicationStreamBatch
+} from './MongoReplicationStreamTypes.js';
 import {
   ChangeStreamBatch,
   parseChangeDocument,
@@ -8,53 +18,116 @@ import {
   rawChangeStream,
   RawChangeStreamOptions
 } from './RawChangeStream.js';
+import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
 
 /**
- * Ordered adapter output. Progress advances source recovery, not the client-visible checkpoint.
+ * Wraps changes with a single-use iterator. Caches the token and metadata after successful exhaustion.
  */
-export type MongoReplicationStreamItem =
-  | {
-      type: 'change';
-      event: ProjectedChangeStreamDocument;
-      /**
-       * More envelopes remain in the received batch, including ones an adapter may exclude.
-       */
-      hasBufferedChanges: boolean;
-    }
-  | {
-      type: 'progress';
-      /**
-       * Covers only complete events already delivered or intentionally excluded by the adapter.
-       */
-      resumeToken: mongo.ResumeToken;
-      /**
-       * Excluded complete events since the previous progress item; reset after every boundary.
-       */
-      filteredCount: number;
-    };
-
-export interface MongoReplicationStreamOptions {
+export function createMongoReplicationStreamBatch({
+  events,
+  resolveResumeToken,
+  resolveMetadata = () => ({ filteredCount: null })
+}: {
   /**
-   * Inserted after namespace selection and before the final large-event split stage.
+   * Complete changes, optionally produced by a lazy parsing or filtering generator.
    */
-  pipelineStages?: mongo.Document[];
+  events: Iterable<MongoReplicationStreamChange>;
   /**
-   * Request pre-images without overriding the reader's post-image, resume or scope options.
+   * Compute a safe resume token once after successful exhaustion, using the final parser state.
    */
-  imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
-}
-
-export interface MongoReplicationStreamContext {
+  resolveResumeToken: () => mongo.ResumeToken | undefined;
   /**
-   * Open the source reader with the current position, namespace, deadlines and cancellation already bound.
-   * The reader returns parsed, reassembled changes followed by safe progress boundaries. A provider can
-   * add stages here and wrap the result to discard marked events or turn filter exits into deletes.
-   * Supplying this callback keeps cursor setup shared with snapshot barriers and resume validation.
-   *
-   * The default provider simply returns open({}). Wrappers must preserve event/progress ordering and
-   * close the returned iterator on completion, cancellation or failure (for await handles this).
+   * Compute this batch's final statistics after exhaustion. Defaults to no filtering.
    */
-  open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamItem>;
+  resolveMetadata?: () => MongoReplicationStreamBatchMetadata;
+}): MongoReplicationStreamBatch {
+  let started = false;
+  let complete = false;
+  let resumeToken: mongo.ResumeToken | undefined;
+  let metadata: MongoReplicationStreamBatchMetadata;
+  function assertComplete() {
+    // Progress is only safe to read after every event has been consumed and finalization succeeds.
+    if (!complete) {
+      throw new ReplicationAssertionError('Consume all batch events before requesting token or metadata');
+    }
+  }
+  return {
+    events: {
+      [Symbol.iterator](): Iterator<MongoReplicationStreamChange> {
+        // A second pass would repeat parsing/filtering against state that has already advanced.
+        if (started) {
+          throw new ReplicationAssertionError('Batch events can only be consumed once');
+        }
+        started = true;
+        return iterate();
+      }
+    },
+    getResumeToken() {
+      assertComplete();
+      return resumeToken;
+    },
+    get metadata() {
+      assertComplete();
+      return metadata;
+    }
+  };
+
+  function* iterate() {
+    // Delegate lazily; an error or early return skips the finalization below.
+    yield* events;
+    // Cache this batch's final results before later batches can change the parser or filter state.
+    resumeToken = resolveResumeToken();
+    metadata = Object.freeze({ ...resolveMetadata() });
+    complete = true;
+  }
+}
+
+/**
+ * Filter complete data changes while retaining source batch boundaries and recovery bookkeeping.
+ * Return MongoReplicationFilterResult.KEEP to forward a change, EXCLUDE to omit it, or DELETE for an exit.
+ * Synthetic deletes retain the source identity, resume token and transaction metadata.
+ * The callback runs synchronously and lazily; failures and early exits close the underlying stream.
+ */
+export async function* filterMongoReplicationStream({
+  batches,
+  filter
+}: MongoReplicationFilterOptions): AsyncGenerator<MongoReplicationStreamBatch> {
+  for await (const sourceBatch of batches) {
+    let filteredCount = 0;
+    const batch = createMongoReplicationStreamBatch({
+      events: filterEvents(),
+      resolveResumeToken: () => sourceBatch.getResumeToken(),
+      resolveMetadata: () => ({ filteredCount: (sourceBatch.metadata.filteredCount ?? 0) + filteredCount })
+    });
+    yield batch;
+    // Assert that the consumer exhausted this batch before requesting the next one, even if every event
+    // was filtered out. The token value is unused; getResumeToken() throws if consumption is incomplete.
+    batch.getResumeToken();
+
+    function* filterEvents(): Generator<MongoReplicationStreamChange> {
+      for (const change of sourceBatch.events) {
+        const { event } = change;
+        if (
+          (event.operationType != 'insert' &&
+            event.operationType != 'update' &&
+            event.operationType != 'replace' &&
+            event.operationType != 'delete') ||
+          event.ns.coll == CHECKPOINTS_COLLECTION
+        ) {
+          yield change;
+          continue;
+        }
+        const result = filter(event);
+        if (result == MongoReplicationFilterResult.EXCLUDE) {
+          filteredCount++;
+        } else if (result == MongoReplicationFilterResult.DELETE) {
+          yield { ...change, event: { ...event, operationType: 'delete' } };
+        } else {
+          yield change;
+        }
+      }
+    }
+  }
 }
 
 /**
@@ -83,8 +156,19 @@ export function openMongoReplicationStream({
   skipInitialTimestamp?: boolean;
   options: RawChangeStreamOptions;
   onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
-}): AsyncIterableIterator<MongoReplicationStreamItem> {
-  const open: MongoReplicationStreamContext['open'] = ({ pipelineStages = [], imageOptions = {} }) => {
+}): AsyncIterableIterator<MongoReplicationStreamBatch> {
+  const parsing = {
+    defaultSchema: db.databaseName,
+    multipleDatabases: namespaceFilter.multipleDatabases,
+    startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
+    signal: options.signal,
+    logger: options.logger
+  };
+
+  const openRaw = async function* ({
+    pipelineStages = [],
+    imageOptions = {}
+  }: MongoReplicationStreamOptions): AsyncIterableIterator<RawMongoReplicationStreamBatch> {
     const clusterScope = isDocumentDb || namespaceFilter.multipleDatabases;
     const streamOptions: mongo.Document = {
       ...imageOptions,
@@ -105,51 +189,35 @@ export function openMongoReplicationStream({
       pipelineStages,
       includeSplitLargeEvent: !isDocumentDb
     });
-    return readMongoReplicationStream({
-      batches: rawChangeStream(clusterScope ? db.client.db('admin') : db, pipeline, options),
-      defaultSchema: db.databaseName,
-      multipleDatabases: namespaceFilter.multipleDatabases,
-      // An exact token can fall inside a transaction. Later events may share its timestamp and must
-      // still be delivered. Timestamp deduplication applies only to legacy positions without tokens.
-      startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
-      signal: options.signal,
-      logger: options.logger,
-      onBatch
-    });
+    for await (const batch of rawChangeStream(clusterScope ? db.client.db('admin') : db, pipeline, options)) {
+      options.signal?.throwIfAborted();
+      using batchScope = onBatch?.(batch) || undefined;
+      yield { events: batch.events, resumeToken: batch.resumeToken };
+    }
   };
-  return queryProvider.openChangeStream({ open });
+
+  return queryProvider.openChangeStream({
+    open: (options) => parseMongoReplicationStream({ batches: openRaw(options), ...parsing })
+  });
 }
 
 /**
- * Parse envelopes once, retaining raw BSON row bodies and withholding progress during split reassembly.
+ * Creates a stateful parser for one raw MongoDB stream. Parses BSON entries, joins split-event
+ * fragments across batches, normalizes namespaces and skips events covered by legacy timestamps.
+ * After processing a batch, callers can request safe progress. Check completion on natural EOF,
+ * not cancellation.
  */
-export async function* readMongoReplicationStream({
-  batches,
+export function createMongoReplicationStreamParser({
   defaultSchema,
   multipleDatabases,
   startAfter,
   signal,
-  logger,
-  onBatch
-}: {
-  batches: AsyncIterable<ChangeStreamBatch>;
-  defaultSchema: string;
-  multipleDatabases: boolean;
-  startAfter?: mongo.Timestamp | null;
-  signal?: AbortSignal;
-  logger?: Logger;
-  /**
-   * Transport accounting happens before parsing/filtering, including partial fragments and empty batches.
-   */
-  onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
-}): AsyncGenerator<MongoReplicationStreamItem> {
+  logger
+}: MongoReplicationParserOptions) {
   let splitDocument: ProjectedChangeStreamDocument | null = null;
   let flexDbNameWorkaroundLogged = false;
-  for await (const batch of batches) {
-    signal?.throwIfAborted();
-    // The optional scope measures batch processing, excluding the next network wait.
-    using batchScope = onBatch?.(batch) || undefined;
-    for (const [index, rawEvent] of batch.events.entries()) {
+  return {
+    parse(rawEvent: Buffer): ProjectedChangeStreamDocument | null {
       signal?.throwIfAborted();
       let event = parseChangeDocument(rawEvent);
       const fragment = event.splitEvent;
@@ -165,14 +233,18 @@ export async function* readMongoReplicationStream({
           throw new ReplicationAssertionError(`Unexpected splitEvent: ${JSON.stringify(fragment)}`);
         }
         splitDocument = splitDocument == null ? event : Object.assign(splitDocument, event);
-        if (fragment.fragment != fragment.of) continue;
+        if (fragment.fragment != fragment.of) {
+          return null;
+        }
         event = splitDocument;
         splitDocument = null;
       } else if (splitDocument != null) {
         throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
       }
 
-      if (startAfter != null && event.clusterTime?.lte(startAfter)) continue;
+      if (startAfter != null && event.clusterTime?.lte(startAfter)) {
+        return null;
+      }
 
       if (
         !multipleDatabases &&
@@ -189,13 +261,54 @@ export async function* readMongoReplicationStream({
           logger?.warn(`Incorrect DB name in change stream: ${originalDatabase}. Changed to ${defaultSchema}.`);
         }
       }
-      yield { type: 'change', event, hasBufferedChanges: index < batch.events.length - 1 };
-    }
-    if (splitDocument == null) {
-      yield { type: 'progress', resumeToken: batch.resumeToken, filteredCount: 0 };
+      return event;
+    },
+    /**
+     * Returns the given transport token if it is safe to persist, or undefined while a split event is unfinished.
+     */
+    getSafeResumeToken(resumeToken: mongo.ResumeToken) {
+      return splitDocument == null ? resumeToken : undefined;
+    },
+    assertComplete() {
+      if (splitDocument != null) {
+        throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+      }
     }
+  };
+}
+
+/**
+ * Wrap raw batches in lazy event iterables. BSON parsing and split-event assembly happen
+ * as consumers request events, without building a parsed array. After exhaustion, each batch retains
+ * its own resume-token result even when later batches change the parser state.
+ * Consume each batch completely before requesting the next; returning early closes the source.
+ */
+export async function* parseMongoReplicationStream({
+  batches,
+  ...options
+}: MongoReplicationParserOptions & {
+  batches: AsyncIterable<RawMongoReplicationStreamBatch>;
+}): AsyncGenerator<MongoReplicationStreamBatch> {
+  const parser = createMongoReplicationStreamParser(options);
+  for await (const sourceBatch of batches) {
+    options.signal?.throwIfAborted();
+    const batch = createMongoReplicationStreamBatch({
+      events: parseEvents(sourceBatch.events),
+      resolveResumeToken: () => parser.getSafeResumeToken(sourceBatch.resumeToken)
+    });
+    yield batch;
+    // Split-event state is shared across batches, so the consumer must finish this one before the next is read.
+    // This also covers empty batches.
+    batch.getResumeToken();
   }
-  if (splitDocument != null) {
-    throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+  parser.assertComplete();
+
+  function* parseEvents(rawEvents: Buffer[]): Generator<MongoReplicationStreamChange> {
+    for (let index = 0; index < rawEvents.length; index++) {
+      const event = parser.parse(rawEvents[index]);
+      if (event != null) {
+        yield { event, hasBufferedChanges: index < rawEvents.length - 1 };
+      }
+    }
   }
 }

package/src/replication/MongoReplicationStreamTypes.ts

@@ -0,0 +1,169 @@
+import type { mongo } from '@powersync/lib-service-mongodb';
+import type { Logger } from '@powersync/lib-services-framework';
+import type { ProjectedChangeStreamDocument } from './RawChangeStream.js';
+
+/**
+ * An unparsed MongoDB response batch. Consumers parse its entries incrementally and retain unfinished
+ * split events across batches before deciding whether its resume token is safe to persist.
+ */
+export interface RawMongoReplicationStreamBatch {
+  /**
+   * Original BSON entries in source order, including possible split-event fragments.
+   */
+  events: Buffer[];
+  /**
+   * Transport position after this batch. It may cover an incomplete event and is not yet safe progress.
+   */
+  resumeToken: mongo.ResumeToken;
+}
+
+/**
+ * A complete event and its position within the original source batch.
+ */
+export interface MongoReplicationStreamChange {
+  event: ProjectedChangeStreamDocument;
+  /**
+   * MongoDB returned more entries after this event, even if a provider later excludes them or the
+   * parser retains them as unfinished split fragments. For [checkpoint, excluded change], preserve
+   * true on the checkpoint so ChangeStream groups commits instead of treating it as caught up.
+   * Providers must preserve this value when forwarding or transforming events.
+   */
+  hasBufferedChanges: boolean;
+}
+
+/**
+ * Statistics for one fully consumed batch, independent of whether its resume token is safe.
+ */
+export interface MongoReplicationStreamBatchMetadata {
+  /**
+   * Complete events excluded from this batch. Null means no filtering was applied; zero means
+   * filtering was applied but no events were excluded. Counts do not carry across batches.
+   */
+  readonly filteredCount: number | null;
+}
+
+/**
+ * A single-use batch of complete events, parsed or filtered lazily as the consumer iterates.
+ * Exhaust events before reading its resume token, metadata or the next batch.
+ */
+export interface MongoReplicationStreamBatch {
+  /**
+   * Complete changes in source order, parsed or filtered lazily during synchronous iteration.
+   * Consume once and fully. It may be empty after filtering or while assembling a split event.
+   */
+  events: Iterable<MongoReplicationStreamChange>;
+  /**
+   * Returns the safe resume token after events are fully consumed, or undefined while a split event
+   * is unfinished. Throws before exhaustion. The result remains stable after later batches.
+   */
+  getResumeToken(): mongo.ResumeToken | undefined;
+  /**
+   * Final statistics for this batch, available even without a safe resume token.
+   * Throws before events are fully consumed; the result is immutable and cached afterward.
+   */
+  readonly metadata: MongoReplicationStreamBatchMetadata;
+}
+
+/**
+ * Provider-controlled additions to the shared cursor setup. Source scope, resume position, post-image
+ * requirements and cancellation remain controlled by the MongoDB replication module.
+ */
+export interface MongoReplicationStreamOptions {
+  /**
+   * Inserted after namespace selection and before the final large-event split stage.
+   */
+  pipelineStages?: mongo.Document[];
+  /**
+   * Request pre-images without overriding the reader's post-image, resume or scope options.
+   */
+  imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
+}
+
+/**
+ * Bound source opener supplied to a query provider for one stream invocation.
+ * Providers may wrap the lazy event iterables to filter or transform complete changes.
+ */
+export interface MongoReplicationStreamContext {
+  /**
+   * Open batches whose events are parsed lazily, with cursor scope, position, cancellation and
+   * accounting already bound. Each call owns independent split-event assembly and parser state.
+   * Events are complete; progress is withheld while a split event is unfinished.
+   * Close the iterator on completion, cancellation or failure (for await handles this).
+   */
+  open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamBatch>;
+}
+
+/**
+ * Source context for lazy parsing of raw BSON events in default and filtering providers.
+ * These options control namespace normalization, legacy timestamp skipping and cancellation; split-event
+ * state belongs to each parser instance and must not be shared between independently opened streams.
+ */
+export interface MongoReplicationParserOptions {
+  /**
+   * Expected database name when correcting Atlas Flex's prefixed namespaces in single-database mode.
+   */
+  defaultSchema: string;
+  /**
+   * Preserve database names when the stream covers multiple databases instead of normalizing them.
+   */
+  multipleDatabases: boolean;
+  /**
+   * Exclude complete events at or before this legacy resume timestamp. Leave unset for exact-token
+   * resumes, where later events can share the same timestamp, and for snapshot barrier probes.
+   */
+  startAfter?: mongo.Timestamp | null;
+  /**
+   * Abort parsing when the owning replication attempt is cancelled. Resolving a batch's progress is not
+   * interrupted, so consumers must check the signal themselves before persisting a token.
+   */
+  signal?: AbortSignal;
+  /**
+   * Receives the namespace-correction warning, at most once per parser instance.
+   */
+  logger?: Logger;
+}
+
+/**
+ * A complete data change eligible for filtering. Checkpoints and DDL bypass the filter.
+ */
+export type MongoReplicationDataChange = Extract<
+  ProjectedChangeStreamDocument,
+  { operationType: 'insert' | 'update' | 'replace' | 'delete' }
+>;
+
+/**
+ * The action to apply to a complete data change after evaluating filter membership.
+ */
+export enum MongoReplicationFilterResult {
+  /**
+   * Forward the original change.
+   */
+  KEEP = 'keep',
+  /**
+   * Omit the change while retaining its batch's recovery progress.
+   */
+  EXCLUDE = 'exclude',
+  /**
+   * Emit a delete because the document left the filtered set.
+   */
+  DELETE = 'delete'
+}
+
+/**
+ * Synchronously decide how to replicate a complete data change. Checkpoints and DDL bypass this callback.
+ */
+export type MongoReplicationFilter = (event: MongoReplicationDataChange) => MongoReplicationFilterResult;
+
+/**
+ * Source batches and membership decisions for the shared replication filter.
+ */
+export interface MongoReplicationFilterOptions {
+  /**
+   * Ordered source batches whose boundaries and recovery progress must be preserved.
+   */
+  batches: AsyncIterable<MongoReplicationStreamBatch>;
+  /**
+   * Decide whether to keep, exclude or delete each complete data change.
+   */
+  filter: MongoReplicationFilter;
+}