@powersync/service-module-mongodb 0.21.2 → 0.22.0

package/src/test-utils/ChangeStreamTestContext.ts

@@ -0,0 +1,520 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAbortedError } from '@powersync/lib-services-framework';
+import {
+  BucketDataRequest,
+  BucketStorageFactory,
+  createCoreReplicationMetrics,
+  initializeCoreReplicationMetrics,
+  InternalOpId,
+  isBatchEnd,
+  LEGACY_STORAGE_VERSION,
+  OplogEntry,
+  ProtocolOpId,
+  ReplicationCheckpoint,
+  settledPromise,
+  storage,
+  SyncRulesBucketStorage,
+  TestStorageFactory,
+  unsettledPromise,
+  updateSyncRulesFromConfig,
+  utils
+} from '@powersync/service-core';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { setTimeout } from 'node:timers/promises';
+
+import { SentinelLSN } from '../common/SentinelLSN.js';
+import { ChangeStream, ChangeStreamOptions } from '../replication/ChangeStream.js';
+import { MongoManager } from '../replication/MongoManager.js';
+import {
+  createCheckpoint,
+  createSentinelCheckpointLsn,
+  STANDALONE_CHECKPOINT_ID
+} from '../replication/MongoRelation.js';
+import { detectDocumentDb } from '../replication/replication-utils.js';
+import { NormalizedMongoConnectionConfig } from '../types/types.js';
+
+/**
+ * Stream settings callers may override while the context owns connections, storage, metrics and cancellation.
+ */
+export type TestChangeStreamOptions = Omit<ChangeStreamOptions, 'connections' | 'storage' | 'metrics' | 'abort_signal'>;
+
+export interface ChangeStreamTestContextOptions {
+  factory: TestStorageFactory;
+  connectionOptions: NormalizedMongoConnectionConfig;
+  metrics: ChangeStreamOptions['metrics'];
+  /**
+   * Preserve both source data and bucket storage when reopening an interrupted replication attempt.
+   */
+  doNotClear?: boolean;
+  storageVersion?: number;
+  streamOptions?: TestChangeStreamOptions;
+  /**
+   * Defaults to detecting DocumentDB from the source's hello response.
+   */
+  documentDbMode?: boolean;
+}
+
+export interface ChangeStreamTestContextResources {
+  factory: BucketStorageFactory;
+  connectionManager: MongoManager;
+  metrics: ChangeStreamOptions['metrics'];
+  storageVersion?: number;
+  streamOptions?: TestChangeStreamOptions;
+  documentDbMode?: boolean;
+}
+
+/**
+ * Owns source/storage connections and real replication. Callers supply their environment and storage implementation.
+ */
+export class ChangeStreamTestContext implements AsyncDisposable {
+  private _walStream?: ChangeStream;
+  private abortController = new AbortController();
+  private settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
+  private syncRulesContent?: storage.PersistedSyncConfigContent;
+  private readonly replicationLeases = new Map<number, Promise<storage.ReplicationLock>>();
+  public storage?: SyncRulesBucketStorage;
+
+  /**
+   * Open a fresh fixture by default; use doNotClear to resume its persisted source and storage state.
+   */
+  static async open(options: ChangeStreamTestContextOptions): Promise<ChangeStreamTestContext> {
+    return this.openWith({ options, createContext: (resources) => new ChangeStreamTestContext(resources) });
+  }
+
+  /**
+   * Share resource setup and failure cleanup with subclasses that supply their own context and defaults.
+   */
+  protected static async openWith<T extends ChangeStreamTestContext>({
+    options,
+    createContext
+  }: {
+    options: ChangeStreamTestContextOptions;
+    createContext: (resources: ChangeStreamTestContextResources) => T | Promise<T>;
+  }): Promise<T> {
+    const factory = await options.factory({ doNotClear: options.doNotClear });
+    try {
+      const connectionManager = new MongoManager(options.connectionOptions);
+      try {
+        if (!options.doNotClear) {
+          await connectionManager.db.dropDatabase();
+        }
+        const documentDbMode = options.documentDbMode ?? (await detectDocumentDb(connectionManager.db));
+        // Await subclass setup so failures still close the shared connections below.
+        return await createContext({ ...options, factory, connectionManager, documentDbMode });
+      } catch (error) {
+        await connectionManager.end();
+        throw error;
+      }
+    } catch (error) {
+      await factory[Symbol.asyncDispose]();
+      throw error;
+    }
+  }
+
+  readonly factory: BucketStorageFactory;
+  readonly connectionManager: MongoManager;
+  protected readonly metrics: ChangeStreamOptions['metrics'];
+  protected readonly streamOptions: TestChangeStreamOptions;
+  protected readonly storageVersion: number;
+  private readonly documentDbMode: boolean;
+
+  /**
+   * Takes ownership of an already configured storage factory and source connection.
+   */
+  constructor({
+    factory,
+    connectionManager,
+    metrics,
+    streamOptions = {},
+    storageVersion = LEGACY_STORAGE_VERSION,
+    documentDbMode = false
+  }: ChangeStreamTestContextResources) {
+    this.factory = factory;
+    this.connectionManager = connectionManager;
+    this.metrics = metrics;
+    this.streamOptions = streamOptions;
+    this.storageVersion = storageVersion;
+    this.documentDbMode = documentDbMode;
+    createCoreReplicationMetrics(metrics);
+    initializeCoreReplicationMetrics(metrics);
+    const unregister = factory.registerListener({
+      beforeDispose: async () => {
+        unregister();
+        try {
+          for (const lease of this.replicationLeases.values()) {
+            await (await lease).release();
+          }
+        } finally {
+          this.replicationLeases.clear();
+        }
+      }
+    });
+  }
+
+  /**
+   * Abort snapshot and/or replication, without actively closing connections.
+   */
+  abort(cause?: Error) {
+    this.abortController.abort(cause);
+  }
+
+  /**
+   * Drain replication before inspecting its final persisted state or closing connections.
+   */
+  async stop(cause?: Error) {
+    this.abort(cause);
+    await this.settledReplicationPromise;
+  }
+
+  async dispose() {
+    await this.stop(new Error('Disposing test context'));
+    try {
+      await this.factory[Symbol.asyncDispose]();
+    } finally {
+      await this.connectionManager.end();
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.dispose();
+  }
+
+  get client() {
+    return this.connectionManager.client;
+  }
+
+  get db() {
+    return this.connectionManager.db;
+  }
+
+  get connectionTag() {
+    return this.connectionManager.connectionTag;
+  }
+
+  async updateSyncRules(content: string) {
+    const replicationStream = await this.factory.updateSyncRules(
+      updateSyncRulesFromConfig(
+        SqlSyncRules.fromYaml(content, {
+          defaultSchema: this.db.databaseName,
+          throwOnError: true
+        }),
+        { storageVersion: this.storageVersion, defaultSchema: this.db.databaseName }
+      )
+    );
+    this.syncRulesContent = replicationStream.syncConfigContent[0];
+    this.storage = await this.getWritableStorage(replicationStream);
+    return this.storage!;
+  }
+
+  async loadNextSyncRules() {
+    const syncConfig = await this.factory.getDeployingSyncConfig();
+    if (syncConfig == null) {
+      throw new Error(`Next sync config not available`);
+    }
+
+    this.syncRulesContent = syncConfig.content;
+    this.storage = await this.getWritableStorage(syncConfig.replicationStream);
+    return this.storage!;
+  }
+
+  async loadActiveSyncRules() {
+    const syncConfig = await this.factory.getActiveSyncConfig();
+    if (syncConfig == null) {
+      throw new Error(`Active sync config not found`);
+    }
+
+    this.syncRulesContent = syncConfig.content;
+    this.storage = await this.getWritableStorage(syncConfig.replicationStream);
+    return this.storage!;
+  }
+
+  /**
+   * Reuse a replication lease per stream until the owning factory is disposed.
+   */
+  private async getWritableStorage(stream: storage.PersistedReplicationStream): Promise<SyncRulesBucketStorage> {
+    let pending = this.replicationLeases.get(stream.replicationStreamId);
+    if (pending == null) {
+      pending = stream.lock();
+      this.replicationLeases.set(stream.replicationStreamId, pending);
+    }
+    let replicationLock: storage.ReplicationLock;
+    try {
+      replicationLock = await pending;
+    } catch (error) {
+      this.replicationLeases.delete(stream.replicationStreamId);
+      throw error;
+    }
+    return this.factory.getInstance(stream, { replicationLock });
+  }
+
+  getSyncConfigContent(): storage.PersistedSyncConfigContent {
+    if (this.syncRulesContent == null) {
+      throw new Error('Sync config not configured - call updateSyncRules() first');
+    }
+    return this.syncRulesContent;
+  }
+
+  get streamer() {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+    if (this._walStream) {
+      return this._walStream;
+    }
+    const options: ChangeStreamOptions = {
+      // Forward all supported stream options, including custom query providers and snapshot/storage hooks.
+      ...this.streamOptions,
+      storage: this.storage,
+      metrics: this.metrics,
+      connections: this.connectionManager,
+      abort_signal: this.abortController.signal,
+      // A shorter await also keeps aborts responsive on MongoDB <= 6.0.
+      maxAwaitTimeMS: this.streamOptions.maxAwaitTimeMS ?? 200
+    };
+    this._walStream = new ChangeStream(options);
+    return this._walStream!;
+  }
+
+  async replicateSnapshot() {
+    this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
+    try {
+      await Promise.race([unsettledPromise(this.settledReplicationPromise), this.streamer.waitForInitialSnapshot()]);
+    } catch (e) {
+      if (e instanceof ReplicationAbortedError && e.cause != null) {
+        throw e.cause;
+      }
+      throw e;
+    }
+  }
+
+  /**
+   * A snapshot is not consistent until streaming replication has caught up.
+   * We simulate that for tests.
+   * Do not use if there are any writes performed while doing the snapshot, as that
+   * would result in inconsistent data.
+   */
+  async markSnapshotConsistent() {
+    let checkpoint: string;
+    if (this.documentDbMode) {
+      const sentinelCheckpoint = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(this.client, this.db));
+      const status = await this.storage!.getStatus();
+      const resumeFrom = status.resumeLsn;
+      const resumeToken = resumeFrom ? SentinelLSN.fromSerialized(resumeFrom).resumeToken : null;
+
+      // This helper artificially marks the snapshot as consistent without
+      // waiting for the stream to observe the sentinel. Keep the sentinel as the
+      // comparable position, but carry forward the existing snapshot resume
+      // token so later DocumentDB streaming still resumes from a real token.
+      checkpoint = new SentinelLSN({
+        sentinel: sentinelCheckpoint.sentinel,
+        resume_token: resumeToken
+      }).comparable;
+    } else {
+      checkpoint = await createCheckpoint(this.db, STANDALONE_CHECKPOINT_ID);
+    }
+
+    await using writer = await this.storage!.createWriter({
+      zeroLSN: '0000000000000000',
+      defaultSchema: this.db.databaseName,
+      storeCurrentData: true
+    });
+    await writer.keepalive(checkpoint);
+    await writer.flush();
+  }
+
+  startStreaming() {
+    this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
+    return this.settledReplicationPromise;
+  }
+
+  /**
+   * Wait for this stream to publish a real source marker; stop polling if replication exits first.
+   */
+  async getCheckpoint(options?: { timeout?: number }): Promise<ReplicationCheckpoint> {
+    if (this.settledReplicationPromise == null) {
+      throw new Error('Start replication before requesting a checkpoint.');
+    }
+    const polling = new AbortController();
+    const wait = getClientCheckpoint({
+      client: this.client,
+      db: this.db,
+      storageFactory: this.factory,
+      storage: this.storage,
+      timeout: options?.timeout ?? 15_000,
+      documentDbMode: this.documentDbMode,
+      signal: polling.signal
+    });
+    try {
+      return await Promise.race([
+        wait,
+        unsettledPromise(this.settledReplicationPromise).then(() => {
+          throw new Error('Replication exited before publishing the requested checkpoint.');
+        })
+      ]);
+    } finally {
+      polling.abort();
+      await wait.catch(() => {});
+    }
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    const map = Object.entries(buckets).map(([bucket, start]) => this.bucketRequest(bucket, start));
+    const chunks = [];
+    for await (const chunk of this.storage!.getBucketDataBatch(checkpoint, map)) {
+      chunks.push(chunk);
+    }
+    return chunks;
+  }
+
+  async getBucketData(bucket: string, start?: ProtocolOpId | InternalOpId | undefined, options?: { timeout?: number }) {
+    const checkpoint = await this.getCheckpoint(options);
+    return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
+  }
+
+  async getBucketDataAtLatestCheckpoint(bucket: string, start?: ProtocolOpId | InternalOpId | undefined) {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+
+    const checkpoint = await this.storage.getCheckpoint();
+    return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
+  }
+
+  async getBucketDataAtCheckpoint(
+    bucket: string,
+    checkpoint: ReplicationCheckpoint,
+    start?: ProtocolOpId | InternalOpId | undefined
+  ) {
+    start ??= 0n;
+    if (typeof start == 'string') {
+      start = BigInt(start);
+    }
+    let map = [this.bucketRequest(bucket, start)];
+    let data: OplogEntry[] = [];
+    while (true) {
+      const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+
+      const chunks = [];
+      for await (const chunk of batch) {
+        chunks.push(chunk);
+      }
+      if (chunks.length == 0) {
+        break;
+      }
+      for (let chunk of chunks) {
+        if (isBatchEnd(chunk)) {
+          if (!chunk.hasMore) {
+            return data;
+          }
+        } else {
+          data = data.concat(chunk.chunkData.data ?? []);
+          map = [this.bucketRequest(bucket, BigInt(chunk.chunkData.next_after))];
+          if (!chunk.chunkData.has_more) {
+            return data;
+          }
+        }
+      }
+    }
+    return data;
+  }
+
+  /**
+   * Resolve a readable test bucket such as global[] to its persisted, versioned bucket name.
+   */
+  private bucketRequest(bucket: string, start: InternalOpId): BucketDataRequest {
+    const parsed = this.getSyncConfigContent().parsed({ defaultSchema: this.db.databaseName });
+    const parameterStart = bucket.indexOf('[');
+    const definitionName = bucket.substring(0, parameterStart);
+    const source = parsed.syncConfigs
+      .flatMap((config) => config.config.bucketDataSources)
+      .find((source) => source.uniqueName == definitionName);
+    if (source == null) {
+      throw new Error(`Unknown bucket '${bucket}'.`);
+    }
+    return {
+      bucket: parsed.hydrationState.getBucketSourceScope(source).bucketPrefix + bucket.substring(parameterStart),
+      source,
+      start
+    };
+  }
+
+  async getChecksums(buckets: string[], options?: { timeout?: number }): Promise<utils.ChecksumMap> {
+    let checkpoint = await this.getCheckpoint(options);
+    const versionedBuckets = buckets.map((bucket) => this.bucketRequest(bucket, 0n));
+    const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
+
+    const unversioned: utils.ChecksumMap = new Map();
+    for (let i = 0; i < buckets.length; i++) {
+      unversioned.set(buckets[i], checksums.get(versionedBuckets[i].bucket)!);
+    }
+    return unversioned;
+  }
+
+  async getChecksum(bucket: string, options?: { timeout?: number }) {
+    const checksums = await this.getChecksums([bucket], options);
+    return checksums.get(bucket);
+  }
+}
+
+export async function getClientCheckpoint({
+  client,
+  db,
+  storageFactory,
+  timeout = 50_000,
+  documentDbMode: forcedDocumentDbMode,
+  storage: expectedStorage,
+  signal
+}: {
+  client: mongo.MongoClient;
+  db: mongo.Db;
+  storageFactory: BucketStorageFactory;
+  timeout?: number;
+  documentDbMode?: boolean;
+  /**
+   * Ignore checkpoints from another active stream while this one is still deploying.
+   */
+  storage?: SyncRulesBucketStorage;
+  signal?: AbortSignal;
+}): Promise<ReplicationCheckpoint> {
+  const start = Date.now();
+  const documentDbMode = forcedDocumentDbMode ?? (await detectDocumentDb(db));
+
+  const lsn = documentDbMode
+    ? await createSentinelCheckpointLsn(client, db)
+    : await createCheckpoint(db, STANDALONE_CHECKPOINT_ID);
+  // The marker establishes a source position. Wait until replication publishes a client-visible checkpoint past it.
+
+  // DocumentDB: the streaming loop skips standalone checkpoint events while a
+  // batch barrier is pending (see ChangeStream.ts), so a single sentinel bump
+  // can be missed on an idle stream with no later event to advance the
+  // checkpoint. Periodically re-bump the sentinel so a standalone event
+  // eventually commits past `lsn`, mirroring getSnapshotLsn's retry loop.
+  const NUDGE_INTERVAL_MS = 1000;
+  let lastNudge = Date.now();
+  let lastCp: ReplicationCheckpoint | null = null;
+
+  while (Date.now() - start < timeout) {
+    signal?.throwIfAborted();
+    const activeStorage = (await storageFactory.getActiveSyncConfig())?.storage;
+    const cp =
+      expectedStorage == null || activeStorage?.replicationStreamId == expectedStorage.replicationStreamId
+        ? await activeStorage?.getCheckpoint()
+        : undefined;
+    if (cp != null) {
+      lastCp = cp;
+      if (cp.lsn && cp.lsn >= lsn) {
+        return cp;
+      }
+    }
+
+    if (documentDbMode && Date.now() - lastNudge >= NUDGE_INTERVAL_MS) {
+      await createSentinelCheckpointLsn(client, db);
+      lastNudge = Date.now();
+    }
+
+    await setTimeout(30, undefined, { signal });
+  }
+
+  throw new Error(`Timeout while waiting for checkpoint ${lsn}. Last checkpoint: ${lastCp?.lsn}`);
+}

package/src/test-utils/test-utils-index.ts

@@ -0,0 +1 @@
+export * from './ChangeStreamTestContext.js';

package/test/src/change_event_memory.test.ts

@@ -0,0 +1,33 @@
+import { parseChangeDocument } from '@module/replication/RawChangeStream.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { expect, test } from 'vitest';
+
+test('parsed document images share the raw event allocation', () => {
+  const raw = createRawEvent('first');
+  const parsed = parseChangeDocument(raw);
+  if (parsed.operationType != 'update') throw new Error('Expected an update');
+
+  for (const image of [parsed.fullDocument, parsed.fullDocumentBeforeChange]) {
+    expect(Buffer.isBuffer(image)).toBe(true);
+    expect(image!.buffer).toBe(raw.buffer);
+    expect(image!.byteOffset).toBeGreaterThan(raw.byteOffset);
+    expect(image!.byteOffset + image!.byteLength).toBeLessThanOrEqual(raw.byteOffset + raw.byteLength);
+  }
+});
+
+/**
+ * Allocate each event independently so Node's shared small-buffer pool cannot explain retained memory.
+ */
+function createRawEvent(id: string): Buffer {
+  const encoded = mongo.BSON.serialize({
+    _id: { _data: id },
+    operationType: 'update',
+    ns: { db: 'app', coll: 'documents' },
+    documentKey: { _id: id },
+    fullDocument: { _id: id, body: 'x'.repeat(16_384) },
+    fullDocumentBeforeChange: { _id: id, body: 'y'.repeat(16_384) }
+  });
+  const raw = Buffer.alloc(encoded.byteLength);
+  raw.set(encoded);
+  return raw;
+}

package/test/src/change_stream.test.ts

@@ -8,7 +8,7 @@ import { test_utils } from '@powersync/service-core-tests';
 
 import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
 import { PostImagesOption } from '@module/types/types.js';
-import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
 import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
 import { testTimeout } from './test-timeouts.js';
 import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
@@ -27,8 +27,8 @@ describe('change stream', () => {
 function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
   const supportsConcurrentSnapshots = storageVersion >= 3;
 
-  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
-    return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
+  const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
+    return openChangeStreamTestContext(factory, { ...options, storageVersion });
   };
   // DocumentDB does not support changeStreamPreAndPostImages, which this test requires via postImages: READ_ONLY.
   test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating basic values', async () => {

package/test/src/change_stream_abort.test.ts

@@ -0,0 +1,110 @@
+import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
+import { createMongoReplicationStreamBatch } from '@module/replication/MongoReplicationStream.js';
+import type { MongoReplicationStreamBatch } from '@module/replication/MongoReplicationStreamTypes.js';
+import { ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { describe, expect, test } from 'vitest';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { describeWithStorage } from './util.js';
+
+type AbortPoint = 'mid-batch' | 'after-last-event';
+
+/**
+ * Abort replication from inside event processing, using real source changes and bucket storage.
+ * The abort fires while the consumer holds an event, so it lands between the parser's own abort checks.
+ */
+describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB change stream abort during a batch', () => {
+  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
+    test.each<AbortPoint>(['mid-batch', 'after-last-event'])('stops replication when aborted %s', async (point) => {
+      {
+        await using context = await openChangeStreamTestContext(factory, { storageVersion });
+        await context.updateSyncRules(/* yaml */ `
+          # Sync config fixture.
+          config:
+            edition: 3
+          streams:
+            global:
+              query: SELECT _id AS id FROM documents
+        `);
+        await context.db.createCollection('documents');
+        await context.replicateSnapshot();
+        await context.getCheckpoint();
+        await context.stop();
+        await context.db
+          .collection<{ _id: number }>('documents')
+          .insertMany(Array.from({ length: 20 }, (_, i) => ({ _id: i + 1 })));
+      }
+
+      const abortCause = new Error('test abort');
+      let abort: (cause: Error) => void = () => {};
+      let triggered = false;
+      let resumeBeforeAbortedBatch: string | null | undefined;
+      let readResumeLsn: () => Promise<string | null | undefined>;
+      let sawDocument = false;
+      const provider = {
+        ...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
+        openChangeStream: ({ open }: Parameters<typeof DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER.openChangeStream>[0]) =>
+          (async function* (): AsyncGenerator<MongoReplicationStreamBatch> {
+            for await (const batch of open({})) {
+              const resumeBeforeBatch = await readResumeLsn();
+              yield createMongoReplicationStreamBatch({
+                events: (function* () {
+                  for (const change of batch.events) {
+                    if ('ns' in change.event && change.event.ns.coll == 'documents') {
+                      sawDocument = true;
+                    }
+                    yield change;
+                    // Runs when the consumer asks for the next event, after it has processed this one.
+                    const atPoint = point == 'mid-batch' ? change.hasBufferedChanges : !change.hasBufferedChanges;
+                    if (!triggered && sawDocument && atPoint) {
+                      triggered = true;
+                      resumeBeforeAbortedBatch = resumeBeforeBatch;
+                      abort(abortCause);
+                    }
+                  }
+                })(),
+                resolveResumeToken: () => batch.getResumeToken(),
+                resolveMetadata: () => batch.metadata
+              });
+            }
+          })()
+      };
+
+      {
+        await using context = await openChangeStreamTestContext(factory, {
+          storageVersion,
+          doNotClear: true,
+          streamOptions: { createReplicationQueryProvider: () => provider }
+        });
+        readResumeLsn = async () => (await context.storage!.getStatus()).resumeLsn;
+        abort = (cause) => context.abort(cause);
+        await context.loadActiveSyncRules();
+        const result = await context.startStreaming();
+
+        expect(triggered).toBe(true);
+        expect(result.status).toBe('rejected');
+        if (result.status != 'rejected') {
+          throw new Error('Expected replication to abort');
+        }
+        if (point == 'after-last-event') {
+          // Every event of the batch was processed, so the consumer's own abort check ends the stream
+          // before it persists the batch's resume token.
+          expect(result.reason).toBeInstanceOf(ReplicationAbortedError);
+        }
+        const cause = result.reason instanceof ReplicationAbortedError ? result.reason.cause : result.reason;
+        expect(cause).toBe(abortCause);
+        expect(resumeBeforeAbortedBatch).toBeTruthy();
+        expect((await context.storage!.getStatus()).resumeLsn).toBe(resumeBeforeAbortedBatch);
+      }
+
+      // Reopen the real storage after the interrupted batch and verify all source rows are recovered.
+      await using resumed = await openChangeStreamTestContext(factory, { storageVersion, doNotClear: true });
+      await resumed.loadActiveSyncRules();
+      resumed.startStreaming();
+      const rows = (await resumed.getBucketData('global|0[]')).filter((op) => op.op == 'PUT');
+      expect([...new Set(rows.map((op) => op.object_id))].sort()).toEqual(
+        Array.from({ length: 20 }, (_, index) => String(index + 1)).sort()
+      );
+    });
+  });
+});