@powersync/service-module-mongodb 0.0.0-dev-20260909133214 → 0.0.0-dev-20260929095716

package/src/replication/ChangeStreamReplicationJob.ts

@@ -3,18 +3,22 @@ import { replication } from '@powersync/service-core';
 
 import { ChangeStream, ChangeStreamInvalidatedError } from './ChangeStream.js';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
+import { MongoReplicationQueryProviderFactory } from './MongoReplicationQueryProvider.js';
 
 export interface ChangeStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions {
   connectionFactory: ConnectionManagerFactory;
+  createReplicationQueryProvider?: MongoReplicationQueryProviderFactory;
 }
 
 export class ChangeStreamReplicationJob extends replication.AbstractReplicationJob {
-  private connectionFactory: ConnectionManagerFactory;
+  private readonly connectionFactory: ConnectionManagerFactory;
+  private readonly createReplicationQueryProvider: MongoReplicationQueryProviderFactory | undefined;
   private lastStream: ChangeStream | null = null;
 
   constructor(options: ChangeStreamReplicationJobOptions) {
     super(options);
     this.connectionFactory = options.connectionFactory;
+    this.createReplicationQueryProvider = options.createReplicationQueryProvider;
     // We use a custom formatter to process the prefix
     this.logger = options.storage.logger;
   }
@@ -72,7 +76,8 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
         metrics: this.options.metrics,
         connections: connectionManager,
         keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
-        logger: this.logger
+        logger: this.logger,
+        createReplicationQueryProvider: this.createReplicationQueryProvider
       });
       this.lastStream = stream;
       await stream.replicate();

package/src/replication/ChangeStreamReplicator.ts

@@ -2,17 +2,21 @@ import { replication, storage } from '@powersync/service-core';
 import { MongoModule } from '../module/MongoModule.js';
 import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
+import { MongoReplicationQueryProviderFactory } from './MongoReplicationQueryProvider.js';
 
 export interface ChangeStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
   connectionFactory: ConnectionManagerFactory;
+  createReplicationQueryProvider?: MongoReplicationQueryProviderFactory;
 }
 
 export class ChangeStreamReplicator extends replication.AbstractReplicator<ChangeStreamReplicationJob> {
   private readonly connectionFactory: ConnectionManagerFactory;
+  private readonly createReplicationQueryProvider: MongoReplicationQueryProviderFactory | undefined;
 
   constructor(options: ChangeStreamReplicatorOptions) {
     super(options);
     this.connectionFactory = options.connectionFactory;
+    this.createReplicationQueryProvider = options.createReplicationQueryProvider;
   }
 
   createJob(options: replication.CreateJobOptions): ChangeStreamReplicationJob {
@@ -21,6 +25,7 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator<Chang
       storage: options.storage,
       metrics: this.metrics,
       connectionFactory: this.connectionFactory,
+      createReplicationQueryProvider: this.createReplicationQueryProvider,
       lock: options.lock,
       rateLimiter: this.rateLimiter
     });

package/src/replication/MongoRelation.ts

@@ -3,11 +3,14 @@ import { storage } from '@powersync/service-core';
 import { JsonContainer } from '@powersync/service-jsonbig';
 import {
   CompatibilityContext,
+  CompatibilityOption,
   CustomArray,
   CustomObject,
   CustomSqliteValue,
   DateTimeSourceOptions,
   DateTimeValue,
+  SQLITE_FALSE,
+  SQLITE_TRUE,
   SqliteInputRow,
   SqliteInputValue,
   TimeValuePrecision
@@ -127,7 +130,11 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
       return data;
     }
   } else if (typeof data == 'boolean') {
-    return data ? 1n : 0n;
+    if (context.isEnabled(CompatibilityOption.fixedBooleanInJson)) {
+      return data;
+    }
+
+    return data ? SQLITE_TRUE : SQLITE_FALSE;
   } else if (typeof data == 'bigint') {
     return data;
   } else if (data instanceof Date) {

package/src/replication/MongoReplicationQueryProvider.ts

@@ -0,0 +1,108 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { SourceTable } from '@powersync/service-core';
+import { HydratedSyncConfig } from '@powersync/service-sync-rules';
+import { MongoManager } from './MongoManager.js';
+import { MongoReplicationStreamContext, MongoReplicationStreamItem } from './MongoReplicationStream.js';
+
+/**
+ * Sync config and source namespace information used to construct a provider for a replication attempt.
+ */
+export interface MongoReplicationQueryProviderContext {
+  /**
+   * Hydrated sync config from which the provider can read its connection-specific options.
+   */
+  syncConfig: HydratedSyncConfig;
+  /**
+   * Identifies the source connection whose options and tables this provider handles.
+   */
+  connectionTag: string;
+  /**
+   * MongoDB database used to resolve table references without an explicit schema.
+   */
+  defaultSchema: string;
+}
+
+/**
+ * Adapts snapshot selection and the stream of source changes for one replication implementation.
+ * The MongoDB module owns cursor options, namespaces, BSON parsing, large-event reassembly and durable progress.
+ */
+export interface MongoReplicationQueryProvider {
+  /**
+   * Validate source capabilities before either snapshot queries or streaming cursors are opened.
+   * Throw if required capabilities are unavailable. Both the stream and snapshotter may call this hook
+   * on the same provider, so validation must be safe to repeat.
+   */
+  validateSource?(options: { connectionManager: MongoManager; isDocumentDb: boolean }): Promise<void>;
+
+  /**
+   * Call context.open() with any additional stages/pre-image options, then optionally wrap its iterator.
+   * Unlike pipeline stages alone, the wrapper can exclude marked events while still forwarding progress.
+   * Yield changes and safe progress, including when all data events are excluded.
+   * Count excluded complete events in the next progress item's filteredCount, resetting it after each
+   * progress item. Preserve the reader's safe resume token rather than advancing past buffered changes.
+   * An exit from a filtered set may be represented as a synthetic delete retaining its original key and token.
+   * Checkpoint/DDL events must remain ordered and intact. Progress alone must never stand in for a checkpoint.
+   * Each call opens a separate iterator, also used for snapshot barriers and resume validation. Keep
+   * per-iterator state separate and close the underlying iterator when the wrapper exits.
+   */
+  openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<MongoReplicationStreamItem>;
+
+  /**
+   * Returns an additional MongoDB find predicate for a physical source table.
+   * Return null to snapshot without an additional filter. Snapshot selection must agree with the
+   * provider's streaming rules so both paths replicate the same set of documents.
+   * Expressions can be supplied as { $expr: ... }. The snapshot query combines the filter with its
+   * _id continuation predicate and uses simple collation whenever a filter is supplied.
+   */
+  getSnapshotFilter(table: SourceTable): mongo.Filter<mongo.Document> | null;
+}
+
+/**
+ * Creates a provider shared by a replication attempt's change stream, snapshotter and source probes.
+ */
+export type MongoReplicationQueryProviderFactory = (
+  context: MongoReplicationQueryProviderContext
+) => MongoReplicationQueryProvider;
+
+/**
+ * Uses the standard MongoDB reader and snapshot selection without additional filtering or event translation.
+ */
+export const DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER: MongoReplicationQueryProvider = {
+  getSnapshotFilter: () => null,
+  openChangeStream: ({ open }) => open({})
+};
+
+/**
+ * Places provider stages after namespace selection and before optional large-event splitting.
+ * This lets provider stages inspect complete events; the shared reader reassembles any resulting fragments.
+ */
+export function buildMongoChangeStreamPipeline(options: {
+  /**
+   * Shared reader options, including resume position, stream scope and requested document images.
+   */
+  streamOptions: mongo.ChangeStreamOptions & mongo.Document;
+  /**
+   * Namespace predicate restricting the stream to the selected source collections and checkpoint events.
+   */
+  namespaceMatch: mongo.Document;
+  /**
+   * Additional aggregation stages supplied by the provider, in execution order.
+   */
+  pipelineStages: mongo.Document[];
+  /**
+   * Append MongoDB's large-event split stage. Disabled for DocumentDB by the shared reader.
+   */
+  includeSplitLargeEvent: boolean;
+}): mongo.Document[] {
+  const pipeline: mongo.Document[] = [
+    { $changeStream: options.streamOptions },
+    { $match: options.namespaceMatch },
+    ...options.pipelineStages
+  ];
+
+  if (options.includeSplitLargeEvent) {
+    pipeline.push({ $changeStreamSplitLargeEvent: {} });
+  }
+
+  return pipeline;
+}

package/src/replication/MongoReplicationStream.ts

@@ -0,0 +1,201 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger, ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { buildMongoChangeStreamPipeline, MongoReplicationQueryProvider } from './MongoReplicationQueryProvider.js';
+import {
+  ChangeStreamBatch,
+  parseChangeDocument,
+  ProjectedChangeStreamDocument,
+  rawChangeStream,
+  RawChangeStreamOptions
+} from './RawChangeStream.js';
+
+/**
+ * Ordered adapter output. Progress advances source recovery, not the client-visible checkpoint.
+ */
+export type MongoReplicationStreamItem =
+  | {
+      type: 'change';
+      event: ProjectedChangeStreamDocument;
+      /**
+       * More envelopes remain in the received batch, including ones an adapter may exclude.
+       */
+      hasBufferedChanges: boolean;
+    }
+  | {
+      type: 'progress';
+      /**
+       * Covers only complete events already delivered or intentionally excluded by the adapter.
+       */
+      resumeToken: mongo.ResumeToken;
+      /**
+       * Excluded complete events since the previous progress item; reset after every boundary.
+       */
+      filteredCount: number;
+    };
+
+export interface MongoReplicationStreamOptions {
+  /**
+   * Inserted after namespace selection and before the final large-event split stage.
+   */
+  pipelineStages?: mongo.Document[];
+  /**
+   * Request pre-images without overriding the reader's post-image, resume or scope options.
+   */
+  imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
+}
+
+export interface MongoReplicationStreamContext {
+  /**
+   * Open the source reader with the current position, namespace, deadlines and cancellation already bound.
+   * The reader returns parsed, reassembled changes followed by safe progress boundaries. A provider can
+   * add stages here and wrap the result to discard marked events or turn filter exits into deletes.
+   * Supplying this callback keeps cursor setup shared with snapshot barriers and resume validation.
+   *
+   * The default provider simply returns open({}). Wrappers must preserve event/progress ordering and
+   * close the returned iterator on completion, cancellation or failure (for await handles this).
+   */
+  open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamItem>;
+}
+
+/**
+ * Shared opening path for ongoing replication, snapshot barriers and resume validation.
+ */
+export function openMongoReplicationStream({
+  db,
+  queryProvider,
+  namespaceFilter,
+  isDocumentDb,
+  usePostImages,
+  position,
+  skipInitialTimestamp = false,
+  options,
+  onBatch
+}: {
+  db: mongo.Db;
+  queryProvider: MongoReplicationQueryProvider;
+  namespaceFilter: { $match: mongo.Document; multipleDatabases: boolean };
+  isDocumentDb: boolean;
+  usePostImages: boolean;
+  position: { resumeAfter?: mongo.ResumeToken | null; startAfter?: mongo.Timestamp | null } | null;
+  /**
+   * Legacy streaming positions exclude the initial timestamp; snapshot barrier probes must include it.
+   */
+  skipInitialTimestamp?: boolean;
+  options: RawChangeStreamOptions;
+  onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
+}): AsyncIterableIterator<MongoReplicationStreamItem> {
+  const open: MongoReplicationStreamContext['open'] = ({ pipelineStages = [], imageOptions = {} }) => {
+    const clusterScope = isDocumentDb || namespaceFilter.multipleDatabases;
+    const streamOptions: mongo.Document = {
+      ...imageOptions,
+      fullDocument: !isDocumentDb && usePostImages ? 'required' : 'updateLookup',
+      ...(!isDocumentDb ? { showExpandedEvents: true } : {}),
+      ...(clusterScope ? { allChangesForCluster: true } : {})
+    };
+    // Only one resume option is legal. Legacy MongoDB snapshot positions may contain just a timestamp;
+    // a fresh DocumentDB stream has neither and opens from now.
+    if (position?.resumeAfter != null) {
+      streamOptions.resumeAfter = position.resumeAfter;
+    } else if (position?.startAfter != null) {
+      streamOptions.startAtOperationTime = position.startAfter;
+    }
+    const pipeline = buildMongoChangeStreamPipeline({
+      streamOptions,
+      namespaceMatch: namespaceFilter.$match,
+      pipelineStages,
+      includeSplitLargeEvent: !isDocumentDb
+    });
+    return readMongoReplicationStream({
+      batches: rawChangeStream(clusterScope ? db.client.db('admin') : db, pipeline, options),
+      defaultSchema: db.databaseName,
+      multipleDatabases: namespaceFilter.multipleDatabases,
+      // An exact token can fall inside a transaction. Later events may share its timestamp and must
+      // still be delivered. Timestamp deduplication applies only to legacy positions without tokens.
+      startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
+      signal: options.signal,
+      logger: options.logger,
+      onBatch
+    });
+  };
+  return queryProvider.openChangeStream({ open });
+}
+
+/**
+ * Parse envelopes once, retaining raw BSON row bodies and withholding progress during split reassembly.
+ */
+export async function* readMongoReplicationStream({
+  batches,
+  defaultSchema,
+  multipleDatabases,
+  startAfter,
+  signal,
+  logger,
+  onBatch
+}: {
+  batches: AsyncIterable<ChangeStreamBatch>;
+  defaultSchema: string;
+  multipleDatabases: boolean;
+  startAfter?: mongo.Timestamp | null;
+  signal?: AbortSignal;
+  logger?: Logger;
+  /**
+   * Transport accounting happens before parsing/filtering, including partial fragments and empty batches.
+   */
+  onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
+}): AsyncGenerator<MongoReplicationStreamItem> {
+  let splitDocument: ProjectedChangeStreamDocument | null = null;
+  let flexDbNameWorkaroundLogged = false;
+  for await (const batch of batches) {
+    signal?.throwIfAborted();
+    // The optional scope measures batch processing, excluding the next network wait.
+    using batchScope = onBatch?.(batch) || undefined;
+    for (const [index, rawEvent] of batch.events.entries()) {
+      signal?.throwIfAborted();
+      let event = parseChangeDocument(rawEvent);
+      const fragment = event.splitEvent;
+      if (fragment != null) {
+        // A retry can resume after a received fragment inside the raw reader. Keep the partial event in
+        // memory across batches/retries, but never expose its token as a durable recovery boundary.
+        const expectedFragment = splitDocument == null ? 1 : splitDocument.splitEvent!.fragment + 1;
+        if (
+          fragment.fragment != expectedFragment ||
+          fragment.fragment > fragment.of ||
+          (splitDocument != null && fragment.of != splitDocument.splitEvent!.of)
+        ) {
+          throw new ReplicationAssertionError(`Unexpected splitEvent: ${JSON.stringify(fragment)}`);
+        }
+        splitDocument = splitDocument == null ? event : Object.assign(splitDocument, event);
+        if (fragment.fragment != fragment.of) continue;
+        event = splitDocument;
+        splitDocument = null;
+      } else if (splitDocument != null) {
+        throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+      }
+
+      if (startAfter != null && event.clusterTime?.lte(startAfter)) continue;
+
+      if (
+        !multipleDatabases &&
+        'ns' in event &&
+        event.ns.db != defaultSchema &&
+        event.ns.db.endsWith(`_${defaultSchema}`)
+      ) {
+        // Atlas Flex can prefix the database name on events recorded while replication was paused.
+        // Normalize before adapters inspect namespaces, just as the ordinary replication loop does.
+        const originalDatabase = event.ns.db;
+        event.ns.db = defaultSchema;
+        if (!flexDbNameWorkaroundLogged) {
+          flexDbNameWorkaroundLogged = true;
+          logger?.warn(`Incorrect DB name in change stream: ${originalDatabase}. Changed to ${defaultSchema}.`);
+        }
+      }
+      yield { type: 'change', event, hasBufferedChanges: index < batch.events.length - 1 };
+    }
+    if (splitDocument == null) {
+      yield { type: 'progress', resumeToken: batch.resumeToken, filteredCount: 0 };
+    }
+  }
+  if (splitDocument != null) {
+    throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+  }
+}

package/src/replication/MongoSnapshotQuery.ts

@@ -15,12 +15,19 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
   private lastCursor: mongo.FindCursor | null = null;
   private collection: mongo.Collection;
   private batchSize: number;
+  private readonly filter: mongo.Filter<mongo.Document> | null;
 
-  public constructor(options: { collection: mongo.Collection; batchSize: number; key?: Uint8Array | null }) {
+  public constructor(options: {
+    collection: mongo.Collection;
+    batchSize: number;
+    key?: Uint8Array | null;
+    filter?: mongo.Filter<mongo.Document> | null;
+  }) {
     this.lastKey = options.key ? bson.deserialize(options.key, { useBigInt64: true })._id : null;
     this.lastCursor = null;
     this.collection = options.collection;
     this.batchSize = options.batchSize;
+    this.filter = options.filter ?? null;
   }
 
   async nextChunk(): Promise<
@@ -39,8 +46,14 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
       // any parsing as an operator.
       // Starting in MongoDB 5.0, this filter can use the _id index. Source:
       // https://www.mongodb.com/docs/manual/release-notes/5.0/#general-aggregation-improvements
-      const filter: mongo.Filter<mongo.Document> =
+      const continuationFilter: mongo.Filter<mongo.Document> =
         this.lastKey == null ? {} : { $expr: { $gt: ['$_id', { $literal: this.lastKey }] } };
+      const filter: mongo.Filter<mongo.Document> =
+        this.filter == null
+          ? continuationFilter
+          : this.lastKey == null
+            ? this.filter
+            : { $and: [this.filter, continuationFilter] };
       cursor = this.collection.find(filter, {
         readConcern: 'majority',
         limit: this.batchSize,
@@ -48,7 +61,10 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
         // See https://github.com/mongodb/node-mongodb-native/pull/4580
         batchSize: this.batchSize + 1,
         sort: { _id: 1 },
-        raw: true
+        raw: true,
+        // Replication predicates have stable binary semantics independent of a
+        // collection's default collation.
+        ...(this.filter == null ? {} : { collation: { locale: 'simple' } })
       });
       newCursor = true;
     }