@powersync/service-module-mongodb 0.0.0-dev-20261007110800 → 0.0.0-dev-20261015132441

package/src/replication/estimateFilteredSnapshotCount.ts

@@ -0,0 +1,93 @@
+import type { mongo } from '@powersync/lib-service-mongodb';
+import type { Logger } from '@powersync/lib-services-framework';
+import { performance } from 'node:perf_hooks';
+
+const COUNT_TIMEOUT_MS = 10_000;
+/**
+ * Keep below 5% for MongoDB's random-cursor optimization on large collections.
+ * https://www.mongodb.com/docs/manual/reference/operator/aggregation/sample/
+ */
+const SAMPLE_FRACTION = 0.01;
+const MAX_SAMPLE_SIZE = 10_000;
+
+interface SampleCounts {
+  sampled: { count: number }[];
+  matches: { count: number }[];
+}
+
+/**
+ * Try an exact filtered count, then estimate selectivity from a bounded sample.
+ * Each attempt has a separate 10-second budget. Failures leave progress unknown (-1).
+ */
+export async function estimateFilteredSnapshotCount(options: {
+  collection: mongo.Collection;
+  filter: mongo.Filter<mongo.Document>;
+  signal: AbortSignal;
+  logger: Logger;
+}): Promise<number> {
+  const { collection, filter, signal, logger } = options;
+  signal.throwIfAborted();
+  try {
+    return await collection.countDocuments(filter, {
+      collation: { locale: 'simple' },
+      readConcern: 'majority',
+      timeoutMS: COUNT_TIMEOUT_MS,
+      signal
+    });
+  } catch (error) {
+    signal.throwIfAborted();
+    logger.debug(`Filtered snapshot count unavailable for ${collection.namespace}; trying a sample strategy`, error);
+  }
+
+  const sampleStartedAt = performance.now();
+  try {
+    const metadataOptions = { timeoutMS: COUNT_TIMEOUT_MS, signal };
+    const totalCount = await collection.estimatedDocumentCount(metadataOptions);
+    signal.throwIfAborted();
+    if (totalCount <= 0) {
+      // Metadata alone cannot establish that a filtered result is empty after the count failed.
+      return -1;
+    }
+    const remainingTimeMS = Math.floor(COUNT_TIMEOUT_MS - (performance.now() - sampleStartedAt));
+    if (remainingTimeMS <= 0) {
+      return -1;
+    }
+    // Keep $sample first and below 5% for MongoDB's random-cursor optimization on large collections.
+    // https://www.mongodb.com/docs/manual/reference/operator/aggregation/sample/
+    const sampleSize = Math.min(MAX_SAMPLE_SIZE, Math.max(1, Math.floor(totalCount * SAMPLE_FRACTION)));
+    const cursor = collection.aggregate<SampleCounts>(
+      [
+        { $sample: { size: sampleSize } },
+        {
+          $facet: {
+            sampled: [{ $count: 'count' }],
+            matches: [{ $match: filter }, { $count: 'count' }]
+          }
+        }
+      ],
+      {
+        collation: { locale: 'simple' },
+        readConcern: 'majority',
+        timeoutMS: remainingTimeMS,
+        signal,
+        allowDiskUse: false
+      }
+    );
+    try {
+      const sample = await cursor.next();
+      const sampledCount = sample?.sampled[0]?.count ?? 0;
+      const matchedCount = sample?.matches[0]?.count ?? 0;
+      if (sampledCount == 0 || matchedCount == 0) {
+        // A rare predicate can miss the whole sample; that does not establish an empty result.
+        return -1;
+      }
+      return Math.round((matchedCount / sampledCount) * totalCount);
+    } finally {
+      await cursor.close();
+    }
+  } catch (error) {
+    signal.throwIfAborted();
+    logger.debug(`Filtered snapshot sample unavailable for ${collection.namespace}; total remains unknown`, error);
+    return -1;
+  }
+}

package/src/replication/replication-index.ts

@@ -7,6 +7,7 @@ export * from './MongoErrorRateLimiter.js';
 export * from './MongoRelation.js';
 export * from './MongoReplicationQueryProvider.js';
 export * from './MongoReplicationStream.js';
+export * from './MongoReplicationStreamTypes.js';
 export * from './MongoSnapshotter.js';
 export * from './RawChangeStream.js';
 export { CHECKPOINTS_COLLECTION } from './replication-utils.js';

package/test/src/change_event_memory.test.ts

@@ -0,0 +1,33 @@
+import { parseChangeDocument } from '@module/replication/RawChangeStream.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { expect, test } from 'vitest';
+
+test('parsed document images share the raw event allocation', () => {
+  const raw = createRawEvent('first');
+  const parsed = parseChangeDocument(raw);
+  if (parsed.operationType != 'update') throw new Error('Expected an update');
+
+  for (const image of [parsed.fullDocument, parsed.fullDocumentBeforeChange]) {
+    expect(Buffer.isBuffer(image)).toBe(true);
+    expect(image!.buffer).toBe(raw.buffer);
+    expect(image!.byteOffset).toBeGreaterThan(raw.byteOffset);
+    expect(image!.byteOffset + image!.byteLength).toBeLessThanOrEqual(raw.byteOffset + raw.byteLength);
+  }
+});
+
+/**
+ * Allocate each event independently so Node's shared small-buffer pool cannot explain retained memory.
+ */
+function createRawEvent(id: string): Buffer {
+  const encoded = mongo.BSON.serialize({
+    _id: { _data: id },
+    operationType: 'update',
+    ns: { db: 'app', coll: 'documents' },
+    documentKey: { _id: id },
+    fullDocument: { _id: id, body: 'x'.repeat(16_384) },
+    fullDocumentBeforeChange: { _id: id, body: 'y'.repeat(16_384) }
+  });
+  const raw = Buffer.alloc(encoded.byteLength);
+  raw.set(encoded);
+  return raw;
+}

package/test/src/change_stream_abort.test.ts

@@ -0,0 +1,110 @@
+import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
+import { createMongoReplicationStreamBatch } from '@module/replication/MongoReplicationStream.js';
+import type { MongoReplicationStreamBatch } from '@module/replication/MongoReplicationStreamTypes.js';
+import { ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { describe, expect, test } from 'vitest';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { describeWithStorage } from './util.js';
+
+type AbortPoint = 'mid-batch' | 'after-last-event';
+
+/**
+ * Abort replication from inside event processing, using real source changes and bucket storage.
+ * The abort fires while the consumer holds an event, so it lands between the parser's own abort checks.
+ */
+describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB change stream abort during a batch', () => {
+  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
+    test.each<AbortPoint>(['mid-batch', 'after-last-event'])('stops replication when aborted %s', async (point) => {
+      {
+        await using context = await openChangeStreamTestContext(factory, { storageVersion });
+        await context.updateSyncRules(/* yaml */ `
+          # Sync config fixture.
+          config:
+            edition: 3
+          streams:
+            global:
+              query: SELECT _id AS id FROM documents
+        `);
+        await context.db.createCollection('documents');
+        await context.replicateSnapshot();
+        await context.getCheckpoint();
+        await context.stop();
+        await context.db
+          .collection<{ _id: number }>('documents')
+          .insertMany(Array.from({ length: 20 }, (_, i) => ({ _id: i + 1 })));
+      }
+
+      const abortCause = new Error('test abort');
+      let abort: (cause: Error) => void = () => {};
+      let triggered = false;
+      let resumeBeforeAbortedBatch: string | null | undefined;
+      let readResumeLsn: () => Promise<string | null | undefined>;
+      let sawDocument = false;
+      const provider = {
+        ...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
+        openChangeStream: ({ open }: Parameters<typeof DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER.openChangeStream>[0]) =>
+          (async function* (): AsyncGenerator<MongoReplicationStreamBatch> {
+            for await (const batch of open({})) {
+              const resumeBeforeBatch = await readResumeLsn();
+              yield createMongoReplicationStreamBatch({
+                events: (function* () {
+                  for (const change of batch.events) {
+                    if ('ns' in change.event && change.event.ns.coll == 'documents') {
+                      sawDocument = true;
+                    }
+                    yield change;
+                    // Runs when the consumer asks for the next event, after it has processed this one.
+                    const atPoint = point == 'mid-batch' ? change.hasBufferedChanges : !change.hasBufferedChanges;
+                    if (!triggered && sawDocument && atPoint) {
+                      triggered = true;
+                      resumeBeforeAbortedBatch = resumeBeforeBatch;
+                      abort(abortCause);
+                    }
+                  }
+                })(),
+                resolveResumeToken: () => batch.getResumeToken(),
+                resolveMetadata: () => batch.metadata
+              });
+            }
+          })()
+      };
+
+      {
+        await using context = await openChangeStreamTestContext(factory, {
+          storageVersion,
+          doNotClear: true,
+          streamOptions: { createReplicationQueryProvider: () => provider }
+        });
+        readResumeLsn = async () => (await context.storage!.getStatus()).resumeLsn;
+        abort = (cause) => context.abort(cause);
+        await context.loadActiveSyncRules();
+        const result = await context.startStreaming();
+
+        expect(triggered).toBe(true);
+        expect(result.status).toBe('rejected');
+        if (result.status != 'rejected') {
+          throw new Error('Expected replication to abort');
+        }
+        if (point == 'after-last-event') {
+          // Every event of the batch was processed, so the consumer's own abort check ends the stream
+          // before it persists the batch's resume token.
+          expect(result.reason).toBeInstanceOf(ReplicationAbortedError);
+        }
+        const cause = result.reason instanceof ReplicationAbortedError ? result.reason.cause : result.reason;
+        expect(cause).toBe(abortCause);
+        expect(resumeBeforeAbortedBatch).toBeTruthy();
+        expect((await context.storage!.getStatus()).resumeLsn).toBe(resumeBeforeAbortedBatch);
+      }
+
+      // Reopen the real storage after the interrupted batch and verify all source rows are recovered.
+      await using resumed = await openChangeStreamTestContext(factory, { storageVersion, doNotClear: true });
+      await resumed.loadActiveSyncRules();
+      resumed.startStreaming();
+      const rows = (await resumed.getBucketData('global|0[]')).filter((op) => op.op == 'PUT');
+      expect([...new Set(rows.map((op) => op.object_id))].sort()).toEqual(
+        Array.from({ length: 20 }, (_, index) => String(index + 1)).sort()
+      );
+    });
+  });
+});

package/test/src/filtering_example.test.ts

@@ -0,0 +1,65 @@
+import {
+  filterMongoReplicationStream,
+  MongoReplicationFilter,
+  MongoReplicationFilterResult,
+  MongoReplicationQueryProvider,
+  parseMongoReplicationStream,
+  RawMongoReplicationStreamBatch
+} from '@module/index.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { expect, test } from 'vitest';
+
+/**
+ * Minimal provider example: use the same immutable ID prefix for snapshots and streaming.
+ * Real BSON parsing and the public filtering helper run without a database connection.
+ */
+test('example: replicate only documents with public IDs and preserve filtered progress', async () => {
+  const filter: MongoReplicationFilter = (event) =>
+    String(event.documentKey._id).startsWith('public-')
+      ? MongoReplicationFilterResult.KEEP
+      : MongoReplicationFilterResult.EXCLUDE;
+  const provider: MongoReplicationQueryProvider = {
+    getSnapshotFilter: () => ({ $expr: { $regexMatch: { input: '$_id', regex: '^public-' } } }),
+    openChangeStream: ({ open }) => filterMongoReplicationStream({ batches: open({}), filter })
+  };
+
+  async function* source(): AsyncGenerator<RawMongoReplicationStreamBatch> {
+    for (const ids of [['public-one', 'private-one', 'public-two'], ['private-two']]) {
+      yield {
+        events: ids.map((_id) =>
+          Buffer.from(
+            mongo.BSON.serialize({
+              _id: { _data: _id },
+              operationType: 'insert',
+              ns: { db: 'app', coll: 'documents' },
+              documentKey: { _id },
+              fullDocument: { _id }
+            })
+          )
+        ),
+        resumeToken: { _data: ids.at(-1) }
+      };
+    }
+  }
+
+  const stream = provider.openChangeStream({
+    open: () => parseMongoReplicationStream({ batches: source(), defaultSchema: 'app', multipleDatabases: false })
+  });
+  const retainedIds = [];
+  const progress = [];
+  for await (const batch of stream) {
+    for (const { event } of batch.events) {
+      if ('documentKey' in event) {
+        retainedIds.push(event.documentKey._id);
+      }
+    }
+    // Read progress only after exhausting the events, including a fully filtered batch.
+    progress.push({ token: batch.getResumeToken(), filteredCount: batch.metadata.filteredCount });
+  }
+
+  expect(retainedIds).toEqual(['public-one', 'public-two']);
+  expect(progress).toEqual([
+    { token: { _data: 'public-two' }, filteredCount: 1 },
+    { token: { _data: 'private-two' }, filteredCount: 1 }
+  ]);
+});

package/test/src/filtering_lifecycle.test.ts

@@ -0,0 +1,101 @@
+import {
+  filterMongoReplicationStream,
+  MongoReplicationFilterResult,
+  MongoReplicationQueryProvider,
+  PostImagesOption
+} from '@module/index.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { describe, expect, test } from 'vitest';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { describeWithStorage } from './util.js';
+
+const CONFIG = /* yaml */ `
+  # Sync config fixture.
+  config:
+    edition: 3
+  streams:
+    global:
+      query: SELECT _id AS id, active FROM documents
+`;
+
+/**
+ * Filter membership is decided by the provider, not by the SQL query, so bucket assertions detect leaks.
+ * Required pre/post-images let the provider distinguish entries, exits and source deletes precisely.
+ */
+const PROVIDER: MongoReplicationQueryProvider = {
+  getSnapshotFilter: () => ({ active: true }),
+  openChangeStream: ({ open }) =>
+    filterMongoReplicationStream({
+      batches: open({ imageOptions: { fullDocumentBeforeChange: 'required' } }),
+      filter: (event) => {
+        const after = 'fullDocument' in event ? event.fullDocument : undefined;
+        const before = 'fullDocumentBeforeChange' in event ? event.fullDocumentBeforeChange : undefined;
+        if (after != null && mongo.BSON.deserialize(after).active === true) {
+          return MongoReplicationFilterResult.KEEP;
+        }
+        if (before != null && mongo.BSON.deserialize(before).active === true) {
+          return MongoReplicationFilterResult.DELETE;
+        }
+        return MongoReplicationFilterResult.EXCLUDE;
+      }
+    })
+};
+
+describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB filtering lifecycle', () => {
+  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
+    test('keeps snapshot and streaming membership consistent across updates, deletes and restart', async () => {
+      const options = {
+        storageVersion,
+        mongoOptions: { postImages: PostImagesOption.READ_ONLY },
+        streamOptions: { createReplicationQueryProvider: () => PROVIDER }
+      };
+      let resumeAfter: bigint;
+      {
+        await using context = await openChangeStreamTestContext(factory, options);
+        await context.updateSyncRules(CONFIG);
+        const collection = await context.db.createCollection<{ _id: string; active: boolean }>('documents', {
+          changeStreamPreAndPostImages: { enabled: true }
+        });
+        await collection.insertMany([
+          { _id: 'one', active: true },
+          { _id: 'two', active: false },
+          { _id: 'three', active: false }
+        ]);
+        await context.replicateSnapshot();
+        let checkpoint = await context.getCheckpoint();
+        expect(await context.getBucketDataAtCheckpoint('global|0[]', checkpoint)).toMatchObject([
+          { op: 'PUT', object_id: 'one' }
+        ]);
+
+        // Enter the selected set through a real update, using the server's write-time post-image.
+        await collection.updateOne({ _id: 'two' }, { $set: { active: true } });
+        expect(await context.getBucketData('global|0[]', checkpoint.checkpoint)).toMatchObject([
+          { op: 'PUT', object_id: 'two' }
+        ]);
+        checkpoint = await context.getCheckpoint();
+
+        // Leaving the set must emit a synthetic delete rather than merely suppressing the update.
+        await collection.updateOne({ _id: 'one' }, { $set: { active: false } });
+        await collection.insertOne({ _id: 'excluded', active: false });
+        expect(await context.getBucketData('global|0[]', checkpoint.checkpoint)).toMatchObject([
+          { op: 'REMOVE', object_id: 'one' }
+        ]);
+        resumeAfter = (await context.getCheckpoint()).checkpoint;
+        await context.stop();
+
+        // Queue a source delete and another entry while replication is stopped.
+        await collection.deleteOne({ _id: 'two' });
+        await collection.updateOne({ _id: 'three' }, { $set: { active: true } });
+      }
+
+      await using resumed = await openChangeStreamTestContext(factory, { ...options, doNotClear: true });
+      await resumed.loadActiveSyncRules();
+      resumed.startStreaming();
+      expect(await resumed.getBucketData('global|0[]', resumeAfter)).toMatchObject([
+        { op: 'REMOVE', object_id: 'two' },
+        { op: 'PUT', object_id: 'three' }
+      ]);
+    });
+  });
+});

package/test/src/query_provider_availability.test.ts

@@ -51,6 +51,7 @@ test.each([
         replicationStreamId: 1,
         replicationStreamName: 'availability',
         getParsedSyncRules: () => hydrated,
+        getParsedSyncConfigSet: () => ({ syncConfigs: [parsed] }),
         reportError,
         logger
       } as unknown as storage.SyncRulesBucketStorage

package/test/src/query_provider_diagnostics.test.ts

@@ -0,0 +1,55 @@
+import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/index.js';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { describe, expect, test } from 'vitest';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
+import { describeWithStorage } from './util.js';
+
+const CONFIG = /* yaml */ `
+  # Sync config fixture.
+  config:
+    edition: 3
+    source_table_options:
+      orders%:
+        mongodb_filter_expression: { $eq: ['$$doc.active', true] }
+      orders:
+        mongodb_filter_expression: disabled
+  streams:
+    orders:
+      query: SELECT _id AS id FROM orders
+`;
+
+describe('MongoDB missing-provider diagnostics', () => {
+  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
+    test('persists a missing-provider error for an active config after the provider is removed', async () => {
+      {
+        await using context = await openChangeStreamTestContext(factory, {
+          storageVersion,
+          streamOptions: { createReplicationQueryProvider: () => DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER }
+        });
+        // The concrete collection opts out, but the wildcard still requires provider support.
+        await context.updateSyncRules(CONFIG);
+        await context.db.createCollection('orders');
+        await context.replicateSnapshot();
+        await context.getCheckpoint();
+      }
+      {
+        await using context = await openChangeStreamTestContext(factory, { storageVersion, doNotClear: true });
+        await context.loadActiveSyncRules();
+        const result = await context.startStreaming();
+        expect(result.status).toBe('rejected');
+        if (result.status != 'rejected') {
+          throw new Error('Expected missing-provider rejection');
+        }
+        expect(result.reason).toBeInstanceOf(ServiceError);
+        expect(result.reason).toMatchObject({ errorData: { code: ErrorCode.PSYNC_S1348 } });
+      }
+      // Reopen storage to prove diagnostics were persisted, rather than only reported in memory.
+      await using reopened = await openChangeStreamTestContext(factory, { storageVersion, doNotClear: true });
+      const active = await reopened.factory.getActiveSyncConfig();
+      expect(active).not.toBeNull();
+      const status = await active!.content.getSyncConfigStatus();
+      expect(status?.last_fatal_error).toContain('no query provider factory is registered');
+      expect(status?.last_fatal_error).toContain('deploy a new sync config');
+    });
+  });
+});