@powersync/service-module-mongodb 0.21.1 → 0.22.0

package/src/replication/ChangeStreamReplicationJob.ts

@@ -3,18 +3,22 @@ import { replication } from '@powersync/service-core';
 
 import { ChangeStream, ChangeStreamInvalidatedError } from './ChangeStream.js';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
+import { MongoReplicationQueryProviderFactory } from './MongoReplicationQueryProvider.js';
 
 export interface ChangeStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions {
   connectionFactory: ConnectionManagerFactory;
+  createReplicationQueryProvider?: MongoReplicationQueryProviderFactory;
 }
 
 export class ChangeStreamReplicationJob extends replication.AbstractReplicationJob {
-  private connectionFactory: ConnectionManagerFactory;
+  private readonly connectionFactory: ConnectionManagerFactory;
+  private readonly createReplicationQueryProvider: MongoReplicationQueryProviderFactory | undefined;
   private lastStream: ChangeStream | null = null;
 
   constructor(options: ChangeStreamReplicationJobOptions) {
     super(options);
     this.connectionFactory = options.connectionFactory;
+    this.createReplicationQueryProvider = options.createReplicationQueryProvider;
     // We use a custom formatter to process the prefix
     this.logger = options.storage.logger;
   }
@@ -66,14 +70,22 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
       if (this.isStopped) {
         return;
       }
-      const stream = new ChangeStream({
-        abort_signal: this.abortController.signal,
-        storage: this.options.storage,
-        metrics: this.options.metrics,
-        connections: connectionManager,
-        keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
-        logger: this.logger
-      });
+      let stream: ChangeStream;
+      try {
+        stream = new ChangeStream({
+          abort_signal: this.abortController.signal,
+          storage: this.options.storage,
+          metrics: this.options.metrics,
+          connections: connectionManager,
+          keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
+          logger: this.logger,
+          createReplicationQueryProvider: this.createReplicationQueryProvider
+        });
+      } catch (error) {
+        // Construction can fail before ChangeStream.replicate() reports errors.
+        await this.storage.reportError(error);
+        throw error;
+      }
       this.lastStream = stream;
       await stream.replicate();
     } finally {

package/src/replication/ChangeStreamReplicator.ts

@@ -2,17 +2,25 @@ import { replication, storage } from '@powersync/service-core';
 import { MongoModule } from '../module/MongoModule.js';
 import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
+import { MongoReplicationQueryProviderFactory } from './MongoReplicationQueryProvider.js';
 
 export interface ChangeStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
   connectionFactory: ConnectionManagerFactory;
+  createReplicationQueryProvider?: MongoReplicationQueryProviderFactory;
 }
 
 export class ChangeStreamReplicator extends replication.AbstractReplicator<ChangeStreamReplicationJob> {
   private readonly connectionFactory: ConnectionManagerFactory;
+  private readonly createReplicationQueryProvider: MongoReplicationQueryProviderFactory | undefined;
 
   constructor(options: ChangeStreamReplicatorOptions) {
     super(options);
     this.connectionFactory = options.connectionFactory;
+    this.createReplicationQueryProvider = options.createReplicationQueryProvider;
+  }
+
+  protected get defaultSchema(): string {
+    return this.connectionFactory.dbConnectionConfig.database;
   }
 
   createJob(options: replication.CreateJobOptions): ChangeStreamReplicationJob {
@@ -21,6 +29,7 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator<Chang
       storage: options.storage,
       metrics: this.metrics,
       connectionFactory: this.connectionFactory,
+      createReplicationQueryProvider: this.createReplicationQueryProvider,
       lock: options.lock,
       rateLimiter: this.rateLimiter
     });

package/src/replication/MongoRelation.ts

@@ -3,11 +3,14 @@ import { storage } from '@powersync/service-core';
 import { JsonContainer } from '@powersync/service-jsonbig';
 import {
   CompatibilityContext,
+  CompatibilityOption,
   CustomArray,
   CustomObject,
   CustomSqliteValue,
   DateTimeSourceOptions,
   DateTimeValue,
+  SQLITE_FALSE,
+  SQLITE_TRUE,
   SqliteInputRow,
   SqliteInputValue,
   TimeValuePrecision
@@ -127,7 +130,11 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
       return data;
     }
   } else if (typeof data == 'boolean') {
-    return data ? 1n : 0n;
+    if (context.isEnabled(CompatibilityOption.fixedBooleanInJson)) {
+      return data;
+    }
+
+    return data ? SQLITE_TRUE : SQLITE_FALSE;
   } else if (typeof data == 'bigint') {
     return data;
   } else if (data instanceof Date) {

package/src/replication/MongoReplicationQueryProvider.ts

@@ -0,0 +1,106 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { SourceTable } from '@powersync/service-core';
+import { HydratedSyncConfig } from '@powersync/service-sync-rules';
+import { MongoManager } from './MongoManager.js';
+import type { MongoReplicationStreamBatch, MongoReplicationStreamContext } from './MongoReplicationStreamTypes.js';
+
+/**
+ * Sync config and source namespace information used to construct a provider for a replication attempt.
+ */
+export interface MongoReplicationQueryProviderContext {
+  /**
+   * Hydrated sync config from which the provider can read its connection-specific options.
+   */
+  syncConfig: HydratedSyncConfig;
+  /**
+   * Identifies the source connection whose options and tables this provider handles.
+   */
+  connectionTag: string;
+  /**
+   * MongoDB database used to resolve table references without an explicit schema.
+   */
+  defaultSchema: string;
+}
+
+/**
+ * Adapts snapshot selection and the stream of source changes for one replication implementation.
+ * The MongoDB module owns cursor options, namespaces, BSON parsing, large-event reassembly and durable progress.
+ */
+export interface MongoReplicationQueryProvider {
+  /**
+   * Validate source capabilities before either snapshot queries or streaming cursors are opened.
+   * Throw if required capabilities are unavailable. Both the stream and snapshotter may call this hook
+   * on the same provider, so validation must be safe to repeat.
+   */
+  validateSource?(options: { connectionManager: MongoManager; isDocumentDb: boolean }): Promise<void>;
+
+  /**
+   * Call context.open() with any additional stages/pre-image options. For filtering, pass its result
+   * to filterMongoReplicationStream({ batches, filter }) with a MongoReplicationFilter callback.
+   * The helper preserves checkpoints, DDL, batch boundaries, buffered-change flags and safe progress,
+   * counts excluded events, and closes the source iterator on failure or early exit.
+   * The default provider returns the opener's batches directly.
+   * Each invocation is independent and also serves snapshot barriers and resume validation; keep
+   * filter state local to that invocation. Snapshot selection must agree with streaming decisions.
+   */
+  openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<MongoReplicationStreamBatch>;
+
+  /**
+   * Returns an additional MongoDB find predicate for a physical source table.
+   * Return null to snapshot without an additional filter. Snapshot selection must agree with the
+   * provider's streaming rules so both paths replicate the same set of documents.
+   * Expressions can be supplied as { $expr: ... }. The snapshot query combines the filter with its
+   * _id continuation predicate and uses simple collation whenever a filter is supplied.
+   */
+  getSnapshotFilter(table: SourceTable): mongo.Filter<mongo.Document> | null;
+}
+
+/**
+ * Creates a provider shared by a replication attempt's change stream, snapshotter and source probes.
+ */
+export type MongoReplicationQueryProviderFactory = (
+  context: MongoReplicationQueryProviderContext
+) => MongoReplicationQueryProvider;
+
+/**
+ * Uses the standard MongoDB reader and snapshot selection without additional filtering or event translation.
+ */
+export const DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER: MongoReplicationQueryProvider = {
+  getSnapshotFilter: () => null,
+  openChangeStream: ({ open }) => open({})
+};
+
+/**
+ * Places provider stages after namespace selection and before optional large-event splitting.
+ * This lets provider stages inspect complete events; the shared reader reassembles any resulting fragments.
+ */
+export function buildMongoChangeStreamPipeline(options: {
+  /**
+   * Shared reader options, including resume position, stream scope and requested document images.
+   */
+  streamOptions: mongo.ChangeStreamOptions & mongo.Document;
+  /**
+   * Namespace predicate restricting the stream to the selected source collections and checkpoint events.
+   */
+  namespaceMatch: mongo.Document;
+  /**
+   * Additional aggregation stages supplied by the provider, in execution order.
+   */
+  pipelineStages: mongo.Document[];
+  /**
+   * Append MongoDB's large-event split stage. Disabled for DocumentDB by the shared reader.
+   */
+  includeSplitLargeEvent: boolean;
+}): mongo.Document[] {
+  const pipeline: mongo.Document[] = [
+    { $changeStream: options.streamOptions },
+    { $match: options.namespaceMatch },
+    ...options.pipelineStages
+  ];
+
+  if (options.includeSplitLargeEvent) {
+    pipeline.push({ $changeStreamSplitLargeEvent: {} });
+  }
+
+  return pipeline;
+}

package/src/replication/MongoReplicationStream.ts

@@ -0,0 +1,314 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { buildMongoChangeStreamPipeline, MongoReplicationQueryProvider } from './MongoReplicationQueryProvider.js';
+import {
+  MongoReplicationFilterOptions,
+  MongoReplicationFilterResult,
+  MongoReplicationParserOptions,
+  MongoReplicationStreamBatch,
+  MongoReplicationStreamBatchMetadata,
+  MongoReplicationStreamChange,
+  MongoReplicationStreamOptions,
+  RawMongoReplicationStreamBatch
+} from './MongoReplicationStreamTypes.js';
+import {
+  ChangeStreamBatch,
+  parseChangeDocument,
+  ProjectedChangeStreamDocument,
+  rawChangeStream,
+  RawChangeStreamOptions
+} from './RawChangeStream.js';
+import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
+
+/**
+ * Wraps changes with a single-use iterator. Caches the token and metadata after successful exhaustion.
+ */
+export function createMongoReplicationStreamBatch({
+  events,
+  resolveResumeToken,
+  resolveMetadata = () => ({ filteredCount: null })
+}: {
+  /**
+   * Complete changes, optionally produced by a lazy parsing or filtering generator.
+   */
+  events: Iterable<MongoReplicationStreamChange>;
+  /**
+   * Compute a safe resume token once after successful exhaustion, using the final parser state.
+   */
+  resolveResumeToken: () => mongo.ResumeToken | undefined;
+  /**
+   * Compute this batch's final statistics after exhaustion. Defaults to no filtering.
+   */
+  resolveMetadata?: () => MongoReplicationStreamBatchMetadata;
+}): MongoReplicationStreamBatch {
+  let started = false;
+  let complete = false;
+  let resumeToken: mongo.ResumeToken | undefined;
+  let metadata: MongoReplicationStreamBatchMetadata;
+  function assertComplete() {
+    // Progress is only safe to read after every event has been consumed and finalization succeeds.
+    if (!complete) {
+      throw new ReplicationAssertionError('Consume all batch events before requesting token or metadata');
+    }
+  }
+  return {
+    events: {
+      [Symbol.iterator](): Iterator<MongoReplicationStreamChange> {
+        // A second pass would repeat parsing/filtering against state that has already advanced.
+        if (started) {
+          throw new ReplicationAssertionError('Batch events can only be consumed once');
+        }
+        started = true;
+        return iterate();
+      }
+    },
+    getResumeToken() {
+      assertComplete();
+      return resumeToken;
+    },
+    get metadata() {
+      assertComplete();
+      return metadata;
+    }
+  };
+
+  function* iterate() {
+    // Delegate lazily; an error or early return skips the finalization below.
+    yield* events;
+    // Cache this batch's final results before later batches can change the parser or filter state.
+    resumeToken = resolveResumeToken();
+    metadata = Object.freeze({ ...resolveMetadata() });
+    complete = true;
+  }
+}
+
+/**
+ * Filter complete data changes while retaining source batch boundaries and recovery bookkeeping.
+ * Return MongoReplicationFilterResult.KEEP to forward a change, EXCLUDE to omit it, or DELETE for an exit.
+ * Synthetic deletes retain the source identity, resume token and transaction metadata.
+ * The callback runs synchronously and lazily; failures and early exits close the underlying stream.
+ */
+export async function* filterMongoReplicationStream({
+  batches,
+  filter
+}: MongoReplicationFilterOptions): AsyncGenerator<MongoReplicationStreamBatch> {
+  for await (const sourceBatch of batches) {
+    let filteredCount = 0;
+    const batch = createMongoReplicationStreamBatch({
+      events: filterEvents(),
+      resolveResumeToken: () => sourceBatch.getResumeToken(),
+      resolveMetadata: () => ({ filteredCount: (sourceBatch.metadata.filteredCount ?? 0) + filteredCount })
+    });
+    yield batch;
+    // Assert that the consumer exhausted this batch before requesting the next one, even if every event
+    // was filtered out. The token value is unused; getResumeToken() throws if consumption is incomplete.
+    batch.getResumeToken();
+
+    function* filterEvents(): Generator<MongoReplicationStreamChange> {
+      for (const change of sourceBatch.events) {
+        const { event } = change;
+        if (
+          (event.operationType != 'insert' &&
+            event.operationType != 'update' &&
+            event.operationType != 'replace' &&
+            event.operationType != 'delete') ||
+          event.ns.coll == CHECKPOINTS_COLLECTION
+        ) {
+          yield change;
+          continue;
+        }
+        const result = filter(event);
+        if (result == MongoReplicationFilterResult.EXCLUDE) {
+          filteredCount++;
+        } else if (result == MongoReplicationFilterResult.DELETE) {
+          yield { ...change, event: { ...event, operationType: 'delete' } };
+        } else {
+          yield change;
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Shared opening path for ongoing replication, snapshot barriers and resume validation.
+ */
+export function openMongoReplicationStream({
+  db,
+  queryProvider,
+  namespaceFilter,
+  isDocumentDb,
+  usePostImages,
+  position,
+  skipInitialTimestamp = false,
+  options,
+  onBatch
+}: {
+  db: mongo.Db;
+  queryProvider: MongoReplicationQueryProvider;
+  namespaceFilter: { $match: mongo.Document; multipleDatabases: boolean };
+  isDocumentDb: boolean;
+  usePostImages: boolean;
+  position: { resumeAfter?: mongo.ResumeToken | null; startAfter?: mongo.Timestamp | null } | null;
+  /**
+   * Legacy streaming positions exclude the initial timestamp; snapshot barrier probes must include it.
+   */
+  skipInitialTimestamp?: boolean;
+  options: RawChangeStreamOptions;
+  onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
+}): AsyncIterableIterator<MongoReplicationStreamBatch> {
+  const parsing = {
+    defaultSchema: db.databaseName,
+    multipleDatabases: namespaceFilter.multipleDatabases,
+    startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
+    signal: options.signal,
+    logger: options.logger
+  };
+
+  const openRaw = async function* ({
+    pipelineStages = [],
+    imageOptions = {}
+  }: MongoReplicationStreamOptions): AsyncIterableIterator<RawMongoReplicationStreamBatch> {
+    const clusterScope = isDocumentDb || namespaceFilter.multipleDatabases;
+    const streamOptions: mongo.Document = {
+      ...imageOptions,
+      fullDocument: !isDocumentDb && usePostImages ? 'required' : 'updateLookup',
+      ...(!isDocumentDb ? { showExpandedEvents: true } : {}),
+      ...(clusterScope ? { allChangesForCluster: true } : {})
+    };
+    // Only one resume option is legal. Legacy MongoDB snapshot positions may contain just a timestamp;
+    // a fresh DocumentDB stream has neither and opens from now.
+    if (position?.resumeAfter != null) {
+      streamOptions.resumeAfter = position.resumeAfter;
+    } else if (position?.startAfter != null) {
+      streamOptions.startAtOperationTime = position.startAfter;
+    }
+    const pipeline = buildMongoChangeStreamPipeline({
+      streamOptions,
+      namespaceMatch: namespaceFilter.$match,
+      pipelineStages,
+      includeSplitLargeEvent: !isDocumentDb
+    });
+    for await (const batch of rawChangeStream(clusterScope ? db.client.db('admin') : db, pipeline, options)) {
+      options.signal?.throwIfAborted();
+      using batchScope = onBatch?.(batch) || undefined;
+      yield { events: batch.events, resumeToken: batch.resumeToken };
+    }
+  };
+
+  return queryProvider.openChangeStream({
+    open: (options) => parseMongoReplicationStream({ batches: openRaw(options), ...parsing })
+  });
+}
+
+/**
+ * Creates a stateful parser for one raw MongoDB stream. Parses BSON entries, joins split-event
+ * fragments across batches, normalizes namespaces and skips events covered by legacy timestamps.
+ * After processing a batch, callers can request safe progress. Check completion on natural EOF,
+ * not cancellation.
+ */
+export function createMongoReplicationStreamParser({
+  defaultSchema,
+  multipleDatabases,
+  startAfter,
+  signal,
+  logger
+}: MongoReplicationParserOptions) {
+  let splitDocument: ProjectedChangeStreamDocument | null = null;
+  let flexDbNameWorkaroundLogged = false;
+  return {
+    parse(rawEvent: Buffer): ProjectedChangeStreamDocument | null {
+      signal?.throwIfAborted();
+      let event = parseChangeDocument(rawEvent);
+      const fragment = event.splitEvent;
+      if (fragment != null) {
+        // A retry can resume after a received fragment inside the raw reader. Keep the partial event in
+        // memory across batches/retries, but never expose its token as a durable recovery boundary.
+        const expectedFragment = splitDocument == null ? 1 : splitDocument.splitEvent!.fragment + 1;
+        if (
+          fragment.fragment != expectedFragment ||
+          fragment.fragment > fragment.of ||
+          (splitDocument != null && fragment.of != splitDocument.splitEvent!.of)
+        ) {
+          throw new ReplicationAssertionError(`Unexpected splitEvent: ${JSON.stringify(fragment)}`);
+        }
+        splitDocument = splitDocument == null ? event : Object.assign(splitDocument, event);
+        if (fragment.fragment != fragment.of) {
+          return null;
+        }
+        event = splitDocument;
+        splitDocument = null;
+      } else if (splitDocument != null) {
+        throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+      }
+
+      if (startAfter != null && event.clusterTime?.lte(startAfter)) {
+        return null;
+      }
+
+      if (
+        !multipleDatabases &&
+        'ns' in event &&
+        event.ns.db != defaultSchema &&
+        event.ns.db.endsWith(`_${defaultSchema}`)
+      ) {
+        // Atlas Flex can prefix the database name on events recorded while replication was paused.
+        // Normalize before adapters inspect namespaces, just as the ordinary replication loop does.
+        const originalDatabase = event.ns.db;
+        event.ns.db = defaultSchema;
+        if (!flexDbNameWorkaroundLogged) {
+          flexDbNameWorkaroundLogged = true;
+          logger?.warn(`Incorrect DB name in change stream: ${originalDatabase}. Changed to ${defaultSchema}.`);
+        }
+      }
+      return event;
+    },
+    /**
+     * Returns the given transport token if it is safe to persist, or undefined while a split event is unfinished.
+     */
+    getSafeResumeToken(resumeToken: mongo.ResumeToken) {
+      return splitDocument == null ? resumeToken : undefined;
+    },
+    assertComplete() {
+      if (splitDocument != null) {
+        throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+      }
+    }
+  };
+}
+
+/**
+ * Wrap raw batches in lazy event iterables. BSON parsing and split-event assembly happen
+ * as consumers request events, without building a parsed array. After exhaustion, each batch retains
+ * its own resume-token result even when later batches change the parser state.
+ * Consume each batch completely before requesting the next; returning early closes the source.
+ */
+export async function* parseMongoReplicationStream({
+  batches,
+  ...options
+}: MongoReplicationParserOptions & {
+  batches: AsyncIterable<RawMongoReplicationStreamBatch>;
+}): AsyncGenerator<MongoReplicationStreamBatch> {
+  const parser = createMongoReplicationStreamParser(options);
+  for await (const sourceBatch of batches) {
+    options.signal?.throwIfAborted();
+    const batch = createMongoReplicationStreamBatch({
+      events: parseEvents(sourceBatch.events),
+      resolveResumeToken: () => parser.getSafeResumeToken(sourceBatch.resumeToken)
+    });
+    yield batch;
+    // Split-event state is shared across batches, so the consumer must finish this one before the next is read.
+    // This also covers empty batches.
+    batch.getResumeToken();
+  }
+  parser.assertComplete();
+
+  function* parseEvents(rawEvents: Buffer[]): Generator<MongoReplicationStreamChange> {
+    for (let index = 0; index < rawEvents.length; index++) {
+      const event = parser.parse(rawEvents[index]);
+      if (event != null) {
+        yield { event, hasBufferedChanges: index < rawEvents.length - 1 };
+      }
+    }
+  }
+}