@powersync/service-module-mongodb 0.21.1 → 0.22.0

package/dist/replication/MongoReplicationStreamTypes.d.ts

@@ -0,0 +1,157 @@
+import type { mongo } from '@powersync/lib-service-mongodb';
+import type { Logger } from '@powersync/lib-services-framework';
+import type { ProjectedChangeStreamDocument } from './RawChangeStream.js';
+/**
+ * An unparsed MongoDB response batch. Consumers parse its entries incrementally and retain unfinished
+ * split events across batches before deciding whether its resume token is safe to persist.
+ */
+export interface RawMongoReplicationStreamBatch {
+    /**
+     * Original BSON entries in source order, including possible split-event fragments.
+     */
+    events: Buffer[];
+    /**
+     * Transport position after this batch. It may cover an incomplete event and is not yet safe progress.
+     */
+    resumeToken: mongo.ResumeToken;
+}
+/**
+ * A complete event and its position within the original source batch.
+ */
+export interface MongoReplicationStreamChange {
+    event: ProjectedChangeStreamDocument;
+    /**
+     * MongoDB returned more entries after this event, even if a provider later excludes them or the
+     * parser retains them as unfinished split fragments. For [checkpoint, excluded change], preserve
+     * true on the checkpoint so ChangeStream groups commits instead of treating it as caught up.
+     * Providers must preserve this value when forwarding or transforming events.
+     */
+    hasBufferedChanges: boolean;
+}
+/**
+ * Statistics for one fully consumed batch, independent of whether its resume token is safe.
+ */
+export interface MongoReplicationStreamBatchMetadata {
+    /**
+     * Complete events excluded from this batch. Null means no filtering was applied; zero means
+     * filtering was applied but no events were excluded. Counts do not carry across batches.
+     */
+    readonly filteredCount: number | null;
+}
+/**
+ * A single-use batch of complete events, parsed or filtered lazily as the consumer iterates.
+ * Exhaust events before reading its resume token, metadata or the next batch.
+ */
+export interface MongoReplicationStreamBatch {
+    /**
+     * Complete changes in source order, parsed or filtered lazily during synchronous iteration.
+     * Consume once and fully. It may be empty after filtering or while assembling a split event.
+     */
+    events: Iterable<MongoReplicationStreamChange>;
+    /**
+     * Returns the safe resume token after events are fully consumed, or undefined while a split event
+     * is unfinished. Throws before exhaustion. The result remains stable after later batches.
+     */
+    getResumeToken(): mongo.ResumeToken | undefined;
+    /**
+     * Final statistics for this batch, available even without a safe resume token.
+     * Throws before events are fully consumed; the result is immutable and cached afterward.
+     */
+    readonly metadata: MongoReplicationStreamBatchMetadata;
+}
+/**
+ * Provider-controlled additions to the shared cursor setup. Source scope, resume position, post-image
+ * requirements and cancellation remain controlled by the MongoDB replication module.
+ */
+export interface MongoReplicationStreamOptions {
+    /**
+     * Inserted after namespace selection and before the final large-event split stage.
+     */
+    pipelineStages?: mongo.Document[];
+    /**
+     * Request pre-images without overriding the reader's post-image, resume or scope options.
+     */
+    imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
+}
+/**
+ * Bound source opener supplied to a query provider for one stream invocation.
+ * Providers may wrap the lazy event iterables to filter or transform complete changes.
+ */
+export interface MongoReplicationStreamContext {
+    /**
+     * Open batches whose events are parsed lazily, with cursor scope, position, cancellation and
+     * accounting already bound. Each call owns independent split-event assembly and parser state.
+     * Events are complete; progress is withheld while a split event is unfinished.
+     * Close the iterator on completion, cancellation or failure (for await handles this).
+     */
+    open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamBatch>;
+}
+/**
+ * Source context for lazy parsing of raw BSON events in default and filtering providers.
+ * These options control namespace normalization, legacy timestamp skipping and cancellation; split-event
+ * state belongs to each parser instance and must not be shared between independently opened streams.
+ */
+export interface MongoReplicationParserOptions {
+    /**
+     * Expected database name when correcting Atlas Flex's prefixed namespaces in single-database mode.
+     */
+    defaultSchema: string;
+    /**
+     * Preserve database names when the stream covers multiple databases instead of normalizing them.
+     */
+    multipleDatabases: boolean;
+    /**
+     * Exclude complete events at or before this legacy resume timestamp. Leave unset for exact-token
+     * resumes, where later events can share the same timestamp, and for snapshot barrier probes.
+     */
+    startAfter?: mongo.Timestamp | null;
+    /**
+     * Abort parsing when the owning replication attempt is cancelled. Resolving a batch's progress is not
+     * interrupted, so consumers must check the signal themselves before persisting a token.
+     */
+    signal?: AbortSignal;
+    /**
+     * Receives the namespace-correction warning, at most once per parser instance.
+     */
+    logger?: Logger;
+}
+/**
+ * A complete data change eligible for filtering. Checkpoints and DDL bypass the filter.
+ */
+export type MongoReplicationDataChange = Extract<ProjectedChangeStreamDocument, {
+    operationType: 'insert' | 'update' | 'replace' | 'delete';
+}>;
+/**
+ * The action to apply to a complete data change after evaluating filter membership.
+ */
+export declare enum MongoReplicationFilterResult {
+    /**
+     * Forward the original change.
+     */
+    KEEP = "keep",
+    /**
+     * Omit the change while retaining its batch's recovery progress.
+     */
+    EXCLUDE = "exclude",
+    /**
+     * Emit a delete because the document left the filtered set.
+     */
+    DELETE = "delete"
+}
+/**
+ * Synchronously decide how to replicate a complete data change. Checkpoints and DDL bypass this callback.
+ */
+export type MongoReplicationFilter = (event: MongoReplicationDataChange) => MongoReplicationFilterResult;
+/**
+ * Source batches and membership decisions for the shared replication filter.
+ */
+export interface MongoReplicationFilterOptions {
+    /**
+     * Ordered source batches whose boundaries and recovery progress must be preserved.
+     */
+    batches: AsyncIterable<MongoReplicationStreamBatch>;
+    /**
+     * Decide whether to keep, exclude or delete each complete data change.
+     */
+    filter: MongoReplicationFilter;
+}

package/dist/replication/MongoReplicationStreamTypes.js

@@ -0,0 +1,19 @@
+/**
+ * The action to apply to a complete data change after evaluating filter membership.
+ */
+export var MongoReplicationFilterResult;
+(function (MongoReplicationFilterResult) {
+    /**
+     * Forward the original change.
+     */
+    MongoReplicationFilterResult["KEEP"] = "keep";
+    /**
+     * Omit the change while retaining its batch's recovery progress.
+     */
+    MongoReplicationFilterResult["EXCLUDE"] = "exclude";
+    /**
+     * Emit a delete because the document left the filtered set.
+     */
+    MongoReplicationFilterResult["DELETE"] = "delete";
+})(MongoReplicationFilterResult || (MongoReplicationFilterResult = {}));
+//# sourceMappingURL=MongoReplicationStreamTypes.js.map

package/dist/replication/MongoReplicationStreamTypes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoReplicationStreamTypes.js","sourceRoot":"","sources":["../../src/replication/MongoReplicationStreamTypes.ts"],"names":[],"mappings":"AAqIA;;GAEG;AACH,MAAM,CAAN,IAAY,4BAaX;AAbD,WAAY,4BAA4B;IACtC;;OAEG;IACH,6CAAa,CAAA;IACb;;OAEG;IACH,mDAAmB,CAAA;IACnB;;OAEG;IACH,iDAAiB,CAAA;AACnB,CAAC,EAbW,4BAA4B,KAA5B,4BAA4B,QAavC"}

package/dist/replication/MongoSnapshotQuery.d.ts

@@ -10,10 +10,12 @@ export declare class ChunkedSnapshotQuery implements AsyncDisposable {
     private lastCursor;
     private collection;
     private batchSize;
+    private readonly filter;
     constructor(options: {
         collection: mongo.Collection;
         batchSize: number;
         key?: Uint8Array | null;
+        filter?: mongo.Filter<mongo.Document> | null;
     });
     nextChunk(): Promise<{
         docs: Buffer[];

package/dist/replication/MongoSnapshotQuery.js

@@ -13,11 +13,13 @@ export class ChunkedSnapshotQuery {
     lastCursor = null;
     collection;
     batchSize;
+    filter;
     constructor(options) {
         this.lastKey = options.key ? bson.deserialize(options.key, { useBigInt64: true })._id : null;
         this.lastCursor = null;
         this.collection = options.collection;
         this.batchSize = options.batchSize;
+        this.filter = options.filter ?? null;
     }
     async nextChunk() {
         let cursor = this.lastCursor;
@@ -33,7 +35,12 @@ export class ChunkedSnapshotQuery {
             // any parsing as an operator.
             // Starting in MongoDB 5.0, this filter can use the _id index. Source:
             // https://www.mongodb.com/docs/manual/release-notes/5.0/#general-aggregation-improvements
-            const filter = this.lastKey == null ? {} : { $expr: { $gt: ['$_id', { $literal: this.lastKey }] } };
+            const continuationFilter = this.lastKey == null ? {} : { $expr: { $gt: ['$_id', { $literal: this.lastKey }] } };
+            const filter = this.filter == null
+                ? continuationFilter
+                : this.lastKey == null
+                    ? this.filter
+                    : { $and: [this.filter, continuationFilter] };
             cursor = this.collection.find(filter, {
                 readConcern: 'majority',
                 limit: this.batchSize,
@@ -41,7 +48,10 @@ export class ChunkedSnapshotQuery {
                 // See https://github.com/mongodb/node-mongodb-native/pull/4580
                 batchSize: this.batchSize + 1,
                 sort: { _id: 1 },
-                raw: true
+                raw: true,
+                // Replication predicates have stable binary semantics independent of a
+                // collection's default collation.
+                ...(this.filter == null ? {} : { collation: { locale: 'simple' } })
             });
             newCursor = true;
         }

package/dist/replication/MongoSnapshotQuery.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoSnapshotQuery.js","sourceRoot":"","sources":["../../src/replication/MongoSnapshotQuery.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAElE;;;;;GAKG;AACH,MAAM,OAAO,oBAAoB;IAC/B,OAAO,GAAQ,IAAI,CAAC;IACZ,UAAU,GAA4B,IAAI,CAAC;IAC3C,UAAU,CAAmB;IAC7B,SAAS,CAAS;IAE1B,YAAmB,OAAqF;QACtG,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7F,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,SAAS;QAGb,IAAI,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACpC,sFAAsF;YACtF,wGAAwG;YACxG,yGAAyG;YACzG,sCAAsC;YACtC,oBAAoB;YACpB,kGAAkG;YAClG,0GAA0G;YAC1G,8BAA8B;YAC9B,sEAAsE;YACtE,0FAA0F;YAC1F,MAAM,MAAM,GACV,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;YACvF,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE;gBACpC,WAAW,EAAE,UAAU;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,SAAS,GAAG,CAAC;gBAC7B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;gBAChB,GAAG,EAAE,IAAI;aACV,CAAC,CAAC;YACH,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,SAAS,EAAE,CAAC;gBACd,8FAA8F;gBAC9F,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,kDAAkD;gBAClD,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,qBAAqB,EAAc,CAAC;QAC5D,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,yBAAyB,CAAC,yEAAyE,CAAC,CAAC;QACjH,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9C,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;IACjC,CAAC;CACF"}
+{"version":3,"file":"MongoSnapshotQuery.js","sourceRoot":"","sources":["../../src/replication/MongoSnapshotQuery.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAElE;;;;;GAKG;AACH,MAAM,OAAO,oBAAoB;IAC/B,OAAO,GAAQ,IAAI,CAAC;IACZ,UAAU,GAA4B,IAAI,CAAC;IAC3C,UAAU,CAAmB;IAC7B,SAAS,CAAS;IACT,MAAM,CAAsC;IAE7D,YAAmB,OAKlB;QACC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7F,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,SAAS;QAGb,IAAI,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACpC,sFAAsF;YACtF,wGAAwG;YACxG,yGAAyG;YACzG,sCAAsC;YACtC,oBAAoB;YACpB,kGAAkG;YAClG,0GAA0G;YAC1G,8BAA8B;YAC9B,sEAAsE;YACtE,0FAA0F;YAC1F,MAAM,kBAAkB,GACtB,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;YACvF,MAAM,MAAM,GACV,IAAI,CAAC,MAAM,IAAI,IAAI;gBACjB,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI;oBACpB,CAAC,CAAC,IAAI,CAAC,MAAM;oBACb,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,EAAE,CAAC;YACpD,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE;gBACpC,WAAW,EAAE,UAAU;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,SAAS,GAAG,CAAC;gBAC7B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;gBAChB,GAAG,EAAE,IAAI;gBACT,uEAAuE;gBACvE,kCAAkC;gBAClC,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;aACpE,CAAC,CAAC;YACH,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,SAAS,EAAE,CAAC;gBACd,8FAA8F;gBAC9F,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,kDAAkD;gBAClD,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,qBAAqB,EAAc,CAAC;QAC5D,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,yBAAyB,CAAC,yEAAyE,CAAC,CAAC;QACjH,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9C,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;IACjC,CAAC;CACF"}

package/dist/replication/MongoSnapshotter.d.ts

@@ -2,6 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
 import { MetricsEngine, SourceTable, storage } from '@powersync/service-core';
 import { MongoManager } from './MongoManager.js';
+import { MongoReplicationQueryProvider } from './MongoReplicationQueryProvider.js';
 export interface MongoSnapshotterOptions {
     connections: MongoManager;
     storage: storage.SyncRulesBucketStorage;
@@ -13,6 +14,7 @@ export interface MongoSnapshotterOptions {
     checkpointStreamId: mongo.ObjectId;
     storageHooks?: storage.StorageHooks;
     snapshotHooks?: MongoSnapshotterHooks;
+    queryProvider?: MongoReplicationQueryProvider;
 }
 export interface MongoSnapshotterHooks {
     beforeSnapshotStarted?: (table: SourceTable) => Promise<void>;
@@ -29,6 +31,7 @@ export declare class MongoSnapshotter {
     private readonly defaultDb;
     private readonly syncRules;
     private readonly sourceRowConverter;
+    private readonly queryProvider;
     private readonly maxAwaitTimeMS;
     private readonly snapshotChunkLength;
     private readonly abortSignal;
@@ -48,7 +51,9 @@ export declare class MongoSnapshotter {
     private get usePostImages();
     private get configurePostImages();
     get supportsConcurrentSnapshots(): boolean;
-    /** The active checkpoint strategy. Only valid after ensureDetected(). */
+    /**
+     * The active checkpoint strategy. Only valid after ensureDetected().
+     */
     private get checkpointImplementation();
     /**
      * Detect DocumentDB and select the checkpoint implementation. Idempotent.
@@ -83,7 +88,7 @@ export declare class MongoSnapshotter {
      */
     private validateSnapshotLsn;
     private getSourceNamespaceFilters;
-    private rawChangeStreamBatches;
+    private openChangeStream;
     private touch;
 }
 export {};

package/dist/replication/MongoSnapshotter.js

@@ -6,9 +6,11 @@ import { MongoLSN } from '../common/MongoLSN.js';
 import { PostImagesOption } from '../types/types.js';
 import { escapeRegExp } from '../utils.js';
 import { createCheckpointImplementation } from './checkpoints/create-checkpoint-implementation.js';
+import { estimateFilteredSnapshotCount } from './estimateFilteredSnapshotCount.js';
 import { getMongoRelation } from './MongoRelation.js';
+import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from './MongoReplicationQueryProvider.js';
+import { openMongoReplicationStream } from './MongoReplicationStream.js';
 import { ChunkedSnapshotQuery } from './MongoSnapshotQuery.js';
-import { parseChangeDocument, rawChangeStream } from './RawChangeStream.js';
 import { CHECKPOINTS_COLLECTION, detectDocumentDb } from './replication-utils.js';
 import { DirectSourceRowConverter } from './SourceRowConverter.js';
 export class MongoSnapshotter {
@@ -19,6 +21,7 @@ export class MongoSnapshotter {
     defaultDb;
     syncRules;
     sourceRowConverter;
+    queryProvider;
     maxAwaitTimeMS;
     snapshotChunkLength;
     abortSignal;
@@ -47,6 +50,7 @@ export class MongoSnapshotter {
         this.checkpointStreamId = options.checkpointStreamId;
         this.storageHooks = options.storageHooks;
         this.snapshotHooks = options.snapshotHooks;
+        this.queryProvider = options.queryProvider ?? DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER;
         this.changeStreamTimeout = Math.ceil(this.client.options.socketTimeoutMS * 0.9);
         this.syncRules = options.storage.getParsedSyncRules({
             defaultSchema: this.defaultDb.databaseName
@@ -67,7 +71,9 @@ export class MongoSnapshotter {
     get supportsConcurrentSnapshots() {
         return this.storage.storageConfig.softDeleteCurrentData;
     }
-    /** The active checkpoint strategy. Only valid after ensureDetected(). */
+    /**
+     * The active checkpoint strategy. Only valid after ensureDetected().
+     */
     get checkpointImplementation() {
         if (this._checkpointImplementation == null) {
             throw new ServiceError(ErrorCode.PSYNC_S1301, 'Checkpoint implementation not initialized - call ensureDetected() first');
@@ -92,6 +98,7 @@ export class MongoSnapshotter {
                 throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
             }
         }
+        await this.queryProvider.validateSource?.({ connectionManager: this.connections, isDocumentDb: this.isDocumentDb });
         this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
             client: this.client,
             db: this.defaultDb,
@@ -181,7 +188,7 @@ export class MongoSnapshotter {
                 totalEstimatedCount: count
             });
             this.queueTable(updated);
-            this.logger.info(`To replicate: ${updated.qualifiedName}: ${updated.snapshotStatus?.replicatedCount}/~${updated.snapshotStatus?.totalEstimatedCount}`);
+            this.logger.info(`To replicate: ${updated.qualifiedName} ${updated.formatSnapshotProgress()}`);
         }
     }
     async waitForInitialSnapshot() {
@@ -364,13 +371,19 @@ export class MongoSnapshotter {
         const rowsReplicatedMetric = this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED);
         const bytesReplicatedMetric = this.metrics.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES);
         const chunksReplicatedMetric = this.metrics.getCounter(ReplicationMetric.CHUNKS_REPLICATED);
-        const totalEstimatedCount = await this.estimatedCountNumber(table);
+        const filter = this.queryProvider.getSnapshotFilter(table);
+        // Reuse known filtered estimates to avoid repeating an expensive count and sample. Unknown totals
+        // (-1) can also represent uninitialized progress, so retry them. Unfiltered metadata counts are cheap
+        // and refreshed before copying rows.
+        const previousEstimate = table.snapshotStatus?.totalEstimatedCount ?? -1;
+        const totalEstimatedCount = filter != null && previousEstimate >= 0 ? previousEstimate : await this.estimatedCountNumber(table, filter);
         let at = table.snapshotStatus?.replicatedCount ?? 0;
         const collection = this.client.db(table.schema).collection(table.name);
         await using query = new ChunkedSnapshotQuery({
             collection,
             key: table.snapshotStatus?.lastKey,
-            batchSize: this.snapshotChunkLength
+            batchSize: this.snapshotChunkLength,
+            filter
         });
         if (query.lastKey != null) {
             this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} - resuming at _id > ${query.lastKey}`);
@@ -445,8 +458,17 @@ export class MongoSnapshotter {
         }
         return result.tables;
     }
-    async estimatedCountNumber(table) {
-        return await this.client.db(table.schema).collection(table.name).estimatedDocumentCount();
+    async estimatedCountNumber(table, filter = this.queryProvider.getSnapshotFilter(table)) {
+        const collection = this.client.db(table.schema).collection(table.name);
+        if (filter != null) {
+            return await estimateFilteredSnapshotCount({
+                collection,
+                filter,
+                signal: this.abortSignal,
+                logger: this.logger
+            });
+        }
+        return await collection.estimatedDocumentCount();
     }
     async getCollectionInfo(db, name) {
         return (await this.client.db(db).listCollections({ name }, { nameOnly: false }).toArray())[0];
@@ -490,7 +512,7 @@ export class MongoSnapshotter {
         this.checkpointImplementation.seedPosition(null);
         const startStreamFromLsn = await this.checkpointImplementation.createFirstBarrier();
         const filters = this.getSourceNamespaceFilters();
-        const iter = this.rawChangeStreamBatches({
+        const iter = this.openChangeStream({
             lsn: startStreamFromLsn,
             maxAwaitTimeMS: 0,
             signal: this.abortSignal,
@@ -500,30 +522,46 @@ export class MongoSnapshotter {
         let lastCheckpointCreated = performance.now();
         let eventsSeen = 0;
         let batchesSeen = 0;
-        for await (const { events } of iter) {
+        // Returns false once the timeout is reached. Otherwise creates another checkpoint when one is due.
+        const shouldKeepWaiting = async () => {
             if (performance.now() - startTime >= LSN_TIMEOUT_SECONDS * 1000) {
-                break;
+                return false;
             }
             if (performance.now() - lastCheckpointCreated >= LSN_CREATE_INTERVAL_SECONDS * 1000) {
                 await this.checkpointImplementation.createBatchCheckpoint();
                 lastCheckpointCreated = performance.now();
             }
-            batchesSeen += 1;
-            for (const rawChangeDocument of events) {
-                const changeDocument = parseChangeDocument(rawChangeDocument);
+            return true;
+        };
+        streamLoop: for await (const sourceBatch of iter) {
+            this.abortSignal.throwIfAborted();
+            for (const { event: changeDocument } of sourceBatch.events) {
+                this.abortSignal.throwIfAborted();
                 const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
                 if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
-                    const kind = this.checkpointImplementation.event.observe(changeDocument);
-                    if (kind != 'own-barrier') {
-                        // Standalone events still feed the implementation's coordinate via
-                        // event.observe above; we only resolve on our own barrier.
-                        continue;
+                    // Standalone events still feed the implementation's coordinate via
+                    // event.observe; we only resolve on our own barrier.
+                    if (this.checkpointImplementation.event.observe(changeDocument) == 'own-barrier') {
+                        // Observing our own barrier has set the coordinate (the barrier event
+                        // carries the sentinel counter directly), so the LSN is ready.
+                        // This is checked before the timeout, so a barrier we already hold is never discarded.
+                        return this.checkpointImplementation.event.lsn(changeDocument);
                     }
-                    // Observing our own barrier has set the coordinate (the barrier event
-                    // carries the sentinel counter directly), so the LSN is ready.
-                    return this.checkpointImplementation.event.lsn(changeDocument);
                 }
-                eventsSeen += 1;
+                else {
+                    eventsSeen += 1;
+                }
+                if (!(await shouldKeepWaiting())) {
+                    break streamLoop;
+                }
+            }
+            const resumeToken = sourceBatch.getResumeToken();
+            if (resumeToken != null) {
+                batchesSeen++;
+            }
+            // Empty batches have no events to trigger the per-event check.
+            if (!(await shouldKeepWaiting())) {
+                break;
             }
         }
         // Could happen if there is a very large replication lag?
@@ -533,13 +571,19 @@ export class MongoSnapshotter {
      * Given a snapshot LSN, validate that we can read from it, by opening a change stream.
      */
     async validateSnapshotLsn(lsn) {
-        const stream = this.rawChangeStreamBatches({
-            lsn,
-            maxAwaitTimeMS: 0,
-            filters: this.getSourceNamespaceFilters()
-        });
-        for await (const _batch of stream) {
-            break;
+        const filters = this.getSourceNamespaceFilters();
+        const stream = this.openChangeStream({ lsn, maxAwaitTimeMS: 0, filters, signal: this.abortSignal });
+        for await (const batch of stream) {
+            this.abortSignal.throwIfAborted();
+            // Reading one complete event validates the position; closing the stream abandons batch progress.
+            for (const _change of batch.events) {
+                return;
+            }
+            // An empty or fully filtered batch can also validate the position through safe progress.
+            // If a split event is unfinished, progress is undefined: keep reading until it is complete.
+            if (batch.getResumeToken() != null) {
+                return;
+            }
         }
     }
     getSourceNamespaceFilters() {
@@ -582,65 +626,23 @@ export class MongoSnapshotter {
         }
         return { $match: nsFilter, multipleDatabases };
     }
-    rawChangeStreamBatches(options) {
-        const position = options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null;
-        const startAfter = position?.startAfter ?? undefined;
-        const resumeAfter = position?.resumeAfter ?? undefined;
-        let fullDocument;
-        if (this.isDocumentDb) {
-            // DocumentDB does not support changeStreamPreAndPostImages, so 'required' won't work.
-            fullDocument = 'updateLookup';
-        }
-        else if (this.usePostImages) {
-            // 'read_only' or 'auto_configure'
-            // Configuration happens during snapshot, or when we see new
-            // collections.
-            fullDocument = 'required';
-        }
-        else {
-            fullDocument = 'updateLookup';
-        }
-        const streamOptions = {
-            fullDocument
-        };
-        if (!this.isDocumentDb) {
-            // DocumentDB does not support showExpandedEvents.
-            streamOptions.showExpandedEvents = true;
-        }
-        const pipeline = [{ $changeStream: streamOptions }, { $match: options.filters.$match }];
-        if (!this.isDocumentDb) {
-            // DocumentDB does not support $changeStreamSplitLargeEvent.
-            pipeline.push({ $changeStreamSplitLargeEvent: {} });
-        }
-        // Only one of these options can be supplied at a time.
-        if (resumeAfter) {
-            streamOptions.resumeAfter = resumeAfter;
-        }
-        else if (startAfter != null) {
-            // Legacy: We don't persist lsns without resumeTokens anymore, but we do still handle the
-            // case if we have an old one. The sentinel implementation never produces a startAfter,
-            // and a fresh DocumentDB stream opens from "now" with neither option set.
-            streamOptions.startAtOperationTime = startAfter;
-        }
-        let watchDb;
-        if (this.isDocumentDb || options.filters.multipleDatabases) {
-            // DocumentDB only supports cluster-level change streams.
-            // Requires readAnyDatabase@admin on Atlas.
-            watchDb = this.client.db('admin');
-            streamOptions.allChangesForCluster = true;
-        }
-        else {
-            // Same general result, but requires less permissions than the above
-            watchDb = this.defaultDb;
-        }
-        const maxAwaitTimeMS = options.maxAwaitTimeMS ?? this.maxAwaitTimeMS;
-        return rawChangeStream(watchDb, pipeline, {
-            batchSize: options.batchSize ?? this.snapshotChunkLength,
-            maxAwaitTimeMS,
-            maxTimeMS: this.changeStreamTimeout,
-            signal: options.signal,
-            logger: this.logger,
-            tracer: options.tracer
+    openChangeStream(options) {
+        return openMongoReplicationStream({
+            db: this.defaultDb,
+            queryProvider: this.queryProvider,
+            namespaceFilter: options.filters,
+            isDocumentDb: this.isDocumentDb,
+            usePostImages: this.usePostImages,
+            position: options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null,
+            onBatch: options.onBatch,
+            options: {
+                batchSize: options.batchSize ?? this.snapshotChunkLength,
+                maxAwaitTimeMS: options.maxAwaitTimeMS ?? this.maxAwaitTimeMS,
+                maxTimeMS: this.changeStreamTimeout,
+                signal: options.signal,
+                logger: this.logger,
+                tracer: options.tracer
+            }
         });
     }
     touch() {