@powersync/service-module-mongodb 0.21.1 → 0.22.0

package/test/src/resume.test.ts

@@ -1,8 +1,9 @@
 import { ChangeStreamInvalidatedError } from '@module/replication/ChangeStream.js';
 import { MongoManager } from '@module/replication/MongoManager.js';
 import { normalizeConnectionConfig } from '@module/types/types.js';
+import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
-import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { ChangeStreamTestContext, openChangeStreamTestContext } from './change_stream_test_setup.js';
 import { env } from './env.js';
 import { describeWithStorage, StorageVersionTestContext } from './util.js';
 
@@ -11,8 +12,8 @@ describe('mongodb resuming replication', () => {
 });
 
 function defineResumeTest({ factory: factoryGenerator, storageVersion }: StorageVersionTestContext) {
-  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
-    return ChangeStreamTestContext.open(factoryGenerator, { ...options, storageVersion });
+  const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
+    return openChangeStreamTestContext(factoryGenerator, { ...options, storageVersion });
   };
 
   test.skip('resuming with a different source database', async () => {
@@ -66,7 +67,12 @@ function defineResumeTest({ factory: factoryGenerator, storageVersion }: Storage
     const factory = await factoryGenerator({ doNotClear: true });
 
     // Create a new context without updating the sync config
-    await using context2 = new ChangeStreamTestContext(factory, connectionManager, {}, storageVersion);
+    await using context2 = new ChangeStreamTestContext({
+      factory,
+      connectionManager,
+      storageVersion,
+      metrics: METRICS_HELPER.metricsEngine
+    });
     const active = await factory.getActiveSyncConfig();
     context2.storage = active!.storage;
 

package/test/src/resuming_snapshots.test.ts

@@ -3,7 +3,7 @@ import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { ReplicationMetric } from '@powersync/service-types';
 import * as timers from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
-import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
 import { env } from './env.js';
 import { describeWithStorage } from './util.js';
 
@@ -35,7 +35,7 @@ async function testResumingReplication(factory: TestStorageFactory, storageVersi
   let startRowCount: number;
 
   {
-    await using context = await ChangeStreamTestContext.open(factory, {
+    await using context = await openChangeStreamTestContext(factory, {
       storageVersion,
       streamOptions: { snapshotChunkLength: 1000 }
     });
@@ -88,7 +88,7 @@ async function testResumingReplication(factory: TestStorageFactory, storageVersi
 
   {
     // Bypass the usual "clear db on factory open" step.
-    await using context2 = await ChangeStreamTestContext.open(factory, {
+    await using context2 = await openChangeStreamTestContext(factory, {
       doNotClear: true,
       storageVersion,
       streamOptions: { snapshotChunkLength: 1000 }

package/test/src/schema.test.ts

@@ -154,27 +154,70 @@ describe('collection schema inference', { timeout: testTimeout(20_000) }, () =>
     expect(executionLimits).toEqual([30_000]);
   });
 
-  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('fails schema inference on a query timeout', async (ctx) => {
-    await db.collection('timeout').insertOne({ value: 'text' });
-    await using adapter = new MongoRouteAPIAdapter({
-      type: 'mongodb',
-      ...TEST_CONNECTION_OPTIONS,
-      database: db.databaseName
-    });
-    await using failCommand = await requireFailCommand(client, ctx);
-    await failCommand.configure({
-      mode: { times: 1 },
-      data: {
-        failCommands: ['aggregate'],
-        errorCode: 50 // MaxTimeMSExpired
-      }
+  // A collection we cannot sample should cost us that collection's fields, not the whole schema.
+  // Exercises the real server/driver error path without waiting for the limits to be reached.
+  describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('when a collection cannot be sampled', () => {
+    // 146 is ExceededMemoryLimit, 292 is QueryExceededMemoryLimitNoDiskUseAllowed, which is what
+    // a small collection of multi-megabyte documents produces when $sample has to sort them.
+    // The namespace filter below needs a server recent enough to honor it (CI runs 6.0+);
+    // older servers ignore it and fail an arbitrary collection's aggregation.
+    test.for([146, 292])('reports it without columns on error %i', async (errorCode, ctx) => {
+      await db.collection('readable').insertOne({ value: 'text' });
+      await db.collection('unreadable').insertOne({ value: 'text' });
+      await using adapter = new MongoRouteAPIAdapter({
+        type: 'mongodb',
+        ...TEST_CONNECTION_OPTIONS,
+        database: db.databaseName
+      });
+      await using failCommand = await requireFailCommand(client, ctx);
+      await failCommand.configure({
+        mode: { times: 1 },
+        data: {
+          failCommands: ['aggregate'],
+          namespace: `${db.databaseName}.unreadable`,
+          errorCode
+        }
+      });
+
+      const schema = await adapter.getConnectionSchema();
+      const tables = schema.find((s) => s.name == db.databaseName)?.tables;
+
+      expect(tables).toEqual(
+        expect.arrayContaining([
+          { name: 'unreadable', columns: [] },
+          expect.objectContaining({
+            name: 'readable',
+            columns: expect.arrayContaining([expect.objectContaining({ name: 'value' })])
+          })
+        ])
+      );
     });
 
-    // Exercise the real server/driver error path without waiting for the full timeout.
-    await expect(adapter.getConnectionSchema()).rejects.toMatchObject({
-      name: 'MongoServerError',
-      code: 50,
-      codeName: 'MaxTimeMSExpired'
+    // 2 is BadValue, standing in for any unexpected error. 50 is MaxTimeMSExpired: timeouts
+    // usually mean the source is overloaded rather than one problem collection, so they keep
+    // failing the request instead of silently degrading every collection.
+    // Neither is retryable, so the driver surfaces them without re-running the aggregation.
+    test.for([2, 50])('still fails the request on error %i', async (errorCode, ctx) => {
+      await db.collection('broken').insertOne({ value: 'text' });
+      await using adapter = new MongoRouteAPIAdapter({
+        type: 'mongodb',
+        ...TEST_CONNECTION_OPTIONS,
+        database: db.databaseName
+      });
+      await using failCommand = await requireFailCommand(client, ctx);
+      await failCommand.configure({
+        mode: { times: 1 },
+        data: {
+          failCommands: ['aggregate'],
+          namespace: `${db.databaseName}.broken`,
+          errorCode
+        }
+      });
+
+      await expect(adapter.getConnectionSchema()).rejects.toMatchObject({
+        name: 'MongoServerError',
+        code: errorCode
+      });
     });
   });
 });

package/test/src/slow_tests.test.ts

@@ -2,7 +2,7 @@ import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js';
+import { openChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_test_setup.js';
 import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
 import { env } from './env.js';
 import { describeWithStorage, StorageVersionTestContext } from './util.js';
@@ -12,8 +12,8 @@ describe.runIf(env.CI || env.SLOW_TESTS)('change stream slow tests', { timeout:
 });
 
 function defineSlowTests({ factory, storageVersion }: StorageVersionTestContext) {
-  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
-    return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
+  const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
+    return openChangeStreamTestContext(factory, { ...options, storageVersion });
   };
 
   // This test uses getParameter/setParameter for minSnapshotHistoryWindowInSeconds,

package/test/src/snapshot_count.test.ts

@@ -0,0 +1,146 @@
+import { estimateFilteredSnapshotCount } from '@module/replication/estimateFilteredSnapshotCount.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { createLogger } from '@powersync/lib-services-framework';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+
+const FILTER = { $expr: { $eq: ['$label', 'live'] } };
+const LOGGER = createLogger({ silent: true });
+
+function fixture() {
+  const client = new mongo.MongoClient('mongodb://localhost:27017');
+  const collection = client.db('snapshot_count_unit').collection('documents');
+  const controller = new AbortController();
+  const count = vi.spyOn(collection, 'countDocuments').mockResolvedValue(42);
+  const metadata = vi.spyOn(collection, 'estimatedDocumentCount').mockResolvedValue(100_000);
+  const cursor = collection.aggregate();
+  const next = vi.spyOn(cursor, 'next').mockResolvedValue({ sampled: [{ count: 1_000 }], matches: [{ count: 250 }] });
+  const close = vi.spyOn(cursor, 'close').mockResolvedValue();
+  const aggregate = vi.spyOn(collection, 'aggregate').mockReturnValue(cursor);
+  const estimate = () =>
+    estimateFilteredSnapshotCount({ collection, filter: FILTER, signal: controller.signal, logger: LOGGER });
+  return { controller, count, metadata, next, close, aggregate, estimate };
+}
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe('filtered snapshot count', () => {
+  test('uses a bounded exact count with the snapshot collation', async () => {
+    const context = fixture();
+    expect(await context.estimate()).toBe(42);
+    expect(context.count).toHaveBeenCalledWith(FILTER, {
+      collation: { locale: 'simple' },
+      readConcern: 'majority',
+      timeoutMS: 10_000,
+      signal: context.controller.signal
+    });
+    expect(context.metadata).not.toHaveBeenCalled();
+    expect(context.aggregate).not.toHaveBeenCalled();
+  });
+
+  test('preserves an exact zero without sampling', async () => {
+    const context = fixture();
+    context.count.mockResolvedValue(0);
+    expect(await context.estimate()).toBe(0);
+    expect(context.metadata).not.toHaveBeenCalled();
+  });
+
+  test('samples before filtering and scales by collection metadata after a count timeout', async () => {
+    const context = fixture();
+    context.count.mockRejectedValue(new mongo.MongoOperationTimeoutError('count timed out'));
+    expect(await context.estimate()).toBe(25_000);
+    expect(context.aggregate).toHaveBeenCalledWith(
+      [
+        { $sample: { size: 1_000 } },
+        { $facet: { sampled: [{ $count: 'count' }], matches: [{ $match: FILTER }, { $count: 'count' }] } }
+      ],
+      expect.objectContaining({
+        collation: { locale: 'simple' },
+        readConcern: 'majority',
+        signal: context.controller.signal,
+        allowDiskUse: false
+      })
+    );
+    const timeoutMS = context.aggregate.mock.calls[0][1]!.timeoutMS!;
+    expect(timeoutMS).toBeGreaterThan(0);
+    expect(timeoutMS).toBeLessThanOrEqual(10_000);
+    expect(context.close).toHaveBeenCalled();
+  });
+
+  test.each([
+    [10, 1],
+    [100_000_000, 10_000]
+  ])('bounds the sample size for a collection of %s documents', async (total, sampleSize) => {
+    const context = fixture();
+    context.count.mockRejectedValue(new Error('count failed'));
+    context.metadata.mockResolvedValue(total);
+    await context.estimate();
+    expect(context.aggregate.mock.calls[0][0]![0]).toEqual({ $sample: { size: sampleSize } });
+  });
+
+  test('uses the actual number sampled when metadata or collection size has changed', async () => {
+    const context = fixture();
+    context.count.mockRejectedValue(new Error('count failed'));
+    context.next.mockResolvedValue({ sampled: [{ count: 80 }], matches: [{ count: 20 }] });
+    expect(await context.estimate()).toBe(25_000);
+  });
+
+  test.each([null, { sampled: [], matches: [] }, { sampled: [{ count: 1_000 }], matches: [] }])(
+    'leaves the total unknown for an empty sample or no matches: %j',
+    async (sample) => {
+      const context = fixture();
+      context.count.mockRejectedValue(new Error('count failed'));
+      context.next.mockResolvedValue(sample);
+      expect(await context.estimate()).toBe(-1);
+      expect(context.close).toHaveBeenCalled();
+    }
+  );
+
+  test.each(['metadata', 'sample', 'empty metadata', 'budget exhausted'])(
+    'leaves the total unknown after %s',
+    async (failure) => {
+      const context = fixture();
+      context.count.mockRejectedValue(new Error('count failed'));
+      if (failure == 'metadata') {
+        context.metadata.mockRejectedValue(new Error('metadata failed'));
+      } else if (failure == 'sample') {
+        context.next.mockRejectedValue(new mongo.MongoOperationTimeoutError('sample timed out'));
+      } else if (failure == 'empty metadata') {
+        context.metadata.mockResolvedValue(0);
+      } else {
+        const { performance } = await import('node:perf_hooks');
+        vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValue(10_001);
+      }
+      expect(await context.estimate()).toBe(-1);
+      if (failure == 'sample') {
+        expect(context.close).toHaveBeenCalled();
+      } else {
+        expect(context.aggregate).not.toHaveBeenCalled();
+      }
+    }
+  );
+
+  test.each(['before count', 'during count', 'during sample'])('propagates cancellation %s', async (when) => {
+    const context = fixture();
+    const reason = new Error('replication aborted');
+    if (when == 'before count') {
+      context.controller.abort(reason);
+    } else if (when == 'during count') {
+      context.count.mockImplementation(async () => {
+        context.controller.abort(reason);
+        throw reason;
+      });
+    } else {
+      context.count.mockRejectedValue(new Error('count failed'));
+      context.next.mockImplementation(async () => {
+        context.controller.abort(reason);
+        throw reason;
+      });
+    }
+    await expect(context.estimate()).rejects.toBe(reason);
+    if (when != 'during sample') {
+      expect(context.metadata).not.toHaveBeenCalled();
+    }
+  });
+});

package/test/src/snapshot_count_integration.test.ts

@@ -0,0 +1,70 @@
+import { estimateFilteredSnapshotCount } from '@module/replication/estimateFilteredSnapshotCount.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { createLogger } from '@powersync/lib-services-framework';
+import { randomUUID } from 'node:crypto';
+import { describe, expect, test, vi } from 'vitest';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { connectMongoData } from './util.js';
+
+const LOGGER = createLogger({ silent: true });
+
+describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('filtered snapshot count integration', () => {
+  test('counts with simple collation even when the collection uses case-insensitive collation', async () => {
+    const { client, db } = await connectMongoData();
+    const collection = await db.createCollection(`snapshot_count_${randomUUID()}`, {
+      collation: { locale: 'en', strength: 2 }
+    });
+    try {
+      await collection.insertMany([{ label: 'live' }, { label: 'LIVE' }, { label: 'other' }]);
+      expect(
+        await estimateFilteredSnapshotCount({
+          collection,
+          filter: { $expr: { $eq: ['$label', 'live'] } },
+          signal: new AbortController().signal,
+          logger: LOGGER
+        })
+      ).toBe(1);
+    } finally {
+      await collection.drop();
+      await client.close();
+    }
+  });
+
+  test('executes the sample pipeline after a failed count and preserves binary filter semantics', async () => {
+    const { client, db } = await connectMongoData();
+    const collection = await db.createCollection(`snapshot_count_${randomUUID()}`, {
+      collation: { locale: 'en', strength: 2 }
+    });
+    try {
+      // For this small collection the sample contains one document. All rows have the same label,
+      // making the sampled result deterministic while exercising the real $sample/$facet pipeline.
+      await collection.insertMany(Array.from({ length: 50 }, () => ({ label: 'live' })));
+      const count = vi
+        .spyOn(collection, 'countDocuments')
+        .mockRejectedValue(new mongo.MongoOperationTimeoutError('timeout'));
+      try {
+        expect(
+          await estimateFilteredSnapshotCount({
+            collection,
+            filter: { $and: [{ label: 'live' }, { $expr: { $eq: ['$label', 'live'] } }] },
+            signal: new AbortController().signal,
+            logger: LOGGER
+          })
+        ).toBe(50);
+        expect(
+          await estimateFilteredSnapshotCount({
+            collection,
+            filter: { $expr: { $eq: ['$label', 'LIVE'] } },
+            signal: new AbortController().signal,
+            logger: LOGGER
+          })
+        ).toBe(-1);
+      } finally {
+        count.mockRestore();
+      }
+    } finally {
+      await collection.drop();
+      await client.close();
+    }
+  });
+});

package/test/src/snapshot_query_provider.test.ts

@@ -0,0 +1,122 @@
+import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { describe, expect, test, vi } from 'vitest';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { describeWithStorage } from './util.js';
+
+describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot query provider', () => {
+  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
+    test.each([
+      { filtered: false, retryUnknown: false },
+      { filtered: true, retryUnknown: false },
+      { filtered: true, retryUnknown: true }
+    ])('pages snapshots: %j', async ({ filtered, retryUnknown }) => {
+      // Run the same snapshot with and without a provider predicate. The filtered case selects even
+      // ids between 2 and 12 whose label is exactly 'live'; null must preserve the full snapshot.
+      const getSnapshotFilter = vi.fn(() =>
+        filtered
+          ? {
+              $expr: {
+                $and: [
+                  { $gt: ['$_id', 2] },
+                  { $lt: ['$_id', 12] },
+                  { $eq: [{ $mod: ['$_id', 2] }, 0] },
+                  { $eq: ['$label', 'live'] }
+                ]
+              }
+            }
+          : null
+      );
+      await using context = await openChangeStreamTestContext(factory, {
+        storageVersion,
+        streamOptions: {
+          snapshotChunkLength: 2,
+          createReplicationQueryProvider: ({ connectionTag, defaultSchema, syncConfig }) => {
+            // Providers need the actual source namespace and parsed source-table options to build their
+            // predicates. This sync config has no source-table options, so those are empty.
+            expect(connectionTag).toBe(context.connectionTag);
+            expect(defaultSchema).toBe(context.db.databaseName);
+            expect(syncConfig.sourceTableConfig).toEqual({});
+            return { ...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER, getSnapshotFilter };
+          }
+        }
+      });
+      await context.updateSyncRules(/* yaml */ `
+        # Sync config fixture.
+        config:
+          edition: 3
+        streams:
+          global:
+            query: SELECT _id AS id, label FROM documents
+      `);
+      // The collection treats 'LIVE' and 'live' as equal by default. Provider filters use simple
+      // collation, so id 6 must be excluded for its 'LIVE' label even though it passes the id conditions.
+      const collection = await context.db.createCollection<{ _id: number; label: string }>('documents', {
+        collation: { locale: 'en', strength: 2 }
+      });
+      await collection.insertMany(
+        Array.from({ length: 14 }, (_, i) => ({ _id: i + 1, label: (i + 1) % 3 == 0 ? 'LIVE' : 'live' }))
+      );
+      const count = vi.spyOn(mongo.Collection.prototype, 'countDocuments');
+      const metadata = vi.spyOn(mongo.Collection.prototype, 'estimatedDocumentCount');
+      try {
+        if (retryUnknown) {
+          // Leave the queued estimate unknown, then let the snapshot retry against the real source.
+          count.mockRejectedValueOnce(new Error('count unavailable'));
+          metadata.mockRejectedValueOnce(new Error('metadata unavailable'));
+        }
+        await context.replicateSnapshot();
+        expect(count).toHaveBeenCalledTimes(retryUnknown ? 2 : filtered ? 1 : 0);
+        expect(metadata).toHaveBeenCalledTimes(retryUnknown ? 1 : filtered ? 0 : 2);
+      } finally {
+        count.mockRestore();
+        metadata.mockRestore();
+      }
+      const rows = (await context.getBucketData('global|0[]')).filter((op) => op.op == 'PUT');
+      // Matching ids 4, 8 and 10 require two pages at a page size of two. After the first page ends
+      // at id 8, pagination adds its own $expr requiring _id > 8. It must combine that condition
+      // with the provider's $expr: id 10 is included, while ids 9 and 11 through 14 remain excluded.
+      // Replacing the provider predicate with the continuation predicate would admit those extra rows.
+      // With no provider filter, pagination must return all 14 documents instead.
+      expect(rows.map((op) => op.object_id).sort()).toEqual(
+        (filtered ? [4, 8, 10] : Array.from({ length: 14 }, (_, i) => i + 1)).map(String).sort()
+      );
+      expect(getSnapshotFilter).toHaveBeenCalledWith(
+        expect.objectContaining({ schema: context.db.databaseName, name: 'documents' })
+      );
+    });
+
+    test('rejects unsupported sources before opening snapshot or streaming queries', async () => {
+      // Simulate a provider requiring a capability the source does not support. Replication must
+      // surface that validation error before asking the provider to open a stream or select snapshot rows.
+      const validateSource = vi.fn(async () => {
+        throw new Error('unsupported test source');
+      });
+      const openChangeStream = vi.fn(DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER.openChangeStream);
+      const getSnapshotFilter = vi.fn(DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER.getSnapshotFilter);
+      await using context = await openChangeStreamTestContext(factory, {
+        storageVersion,
+        streamOptions: {
+          createReplicationQueryProvider: () => ({ validateSource, openChangeStream, getSnapshotFilter })
+        }
+      });
+      await context.updateSyncRules(/* yaml */ `
+        # Sync config fixture.
+        config:
+          edition: 3
+        streams:
+          global:
+            query: SELECT _id AS id FROM documents
+      `);
+      await expect(context.replicateSnapshot()).rejects.toThrow('unsupported test source');
+      expect(validateSource).toHaveBeenCalledWith({
+        connectionManager: context.connectionManager,
+        isDocumentDb: false
+      });
+      // Spies delegate to the real default hooks if invoked, but validation should prevent either call.
+      expect(openChangeStream).not.toHaveBeenCalled();
+      expect(getSnapshotFilter).not.toHaveBeenCalled();
+    });
+  });
+});