@powersync/service-module-mongodb 0.21.2 → 0.22.0

package/test/src/change_stream_test_setup.ts

@@ -0,0 +1,38 @@
+import { test_utils } from '@module/index.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { TestStorageFactory } from '@powersync/service-core';
+import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { TEST_CONNECTION_OPTIONS } from './util.js';
+
+export const ChangeStreamTestContext = test_utils.ChangeStreamTestContext;
+
+/**
+ * Supply this suite's environment and metrics to the exported context.
+ */
+export function openChangeStreamTestContext(
+  factory: TestStorageFactory,
+  options?: Omit<test_utils.ChangeStreamTestContextOptions, 'factory' | 'connectionOptions' | 'metrics'> & {
+    mongoOptions?: Partial<test_utils.ChangeStreamTestContextOptions['connectionOptions']>;
+  }
+) {
+  return ChangeStreamTestContext.open({
+    ...options,
+    factory,
+    connectionOptions: { ...TEST_CONNECTION_OPTIONS, ...options?.mongoOptions },
+    metrics: METRICS_HELPER.metricsEngine
+  });
+}
+
+export async function setSnapshotHistorySeconds(client: mongo.MongoClient, seconds: number) {
+  const { minSnapshotHistoryWindowInSeconds: currentValue } = await client
+    .db('admin')
+    .command({ getParameter: 1, minSnapshotHistoryWindowInSeconds: 1 });
+
+  await client.db('admin').command({ setParameter: 1, minSnapshotHistoryWindowInSeconds: seconds });
+
+  return {
+    async [Symbol.asyncDispose]() {
+      await client.db('admin').command({ setParameter: 1, minSnapshotHistoryWindowInSeconds: currentValue });
+    }
+  };
+}

package/test/src/chunked_snapshot.test.ts

@@ -5,7 +5,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
 import { describeWithStorage, StorageVersionTestContext } from './util.js';
 
 describe('chunked snapshots', () => {
@@ -13,8 +13,8 @@ describe('chunked snapshots', () => {
 });
 
 function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
-  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
-    return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
+  const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
+    return openChangeStreamTestContext(factory, { ...options, storageVersion });
   };
 
   // This is not as sensitive to the id type as postgres, but we still test a couple of cases

package/test/src/documentdb_mode.test.ts

@@ -9,7 +9,7 @@ import { SentinelLSN } from '@module/common/SentinelLSN.js';
 import { createSentinelCheckpointLsn, SENTINEL_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
 import { CHECKPOINTS_COLLECTION } from '@module/replication/replication-utils.js';
 import { mongo } from '@powersync/lib-service-mongodb';
-import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
 import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
 import { env } from './env.js';
 import { testTimeout } from './test-timeouts.js';
@@ -584,8 +584,8 @@ describe.skipIf(DATABASE_TYPE != DatabaseType.DOCUMENTDB)('documentDbMode', () =
 });
 
 function defineDocumentDBDbModeTests({ factory, storageVersion }: StorageVersionTestContext) {
-  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
-    return ChangeStreamTestContext.open(factory, {
+  const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
+    return openChangeStreamTestContext(factory, {
       ...options,
       storageVersion,
       streamOptions: {

package/test/src/filtering_example.test.ts

@@ -0,0 +1,65 @@
+import {
+  filterMongoReplicationStream,
+  MongoReplicationFilter,
+  MongoReplicationFilterResult,
+  MongoReplicationQueryProvider,
+  parseMongoReplicationStream,
+  RawMongoReplicationStreamBatch
+} from '@module/index.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { expect, test } from 'vitest';
+
+/**
+ * Minimal provider example: use the same immutable ID prefix for snapshots and streaming.
+ * Real BSON parsing and the public filtering helper run without a database connection.
+ */
+test('example: replicate only documents with public IDs and preserve filtered progress', async () => {
+  const filter: MongoReplicationFilter = (event) =>
+    String(event.documentKey._id).startsWith('public-')
+      ? MongoReplicationFilterResult.KEEP
+      : MongoReplicationFilterResult.EXCLUDE;
+  const provider: MongoReplicationQueryProvider = {
+    getSnapshotFilter: () => ({ $expr: { $regexMatch: { input: '$_id', regex: '^public-' } } }),
+    openChangeStream: ({ open }) => filterMongoReplicationStream({ batches: open({}), filter })
+  };
+
+  async function* source(): AsyncGenerator<RawMongoReplicationStreamBatch> {
+    for (const ids of [['public-one', 'private-one', 'public-two'], ['private-two']]) {
+      yield {
+        events: ids.map((_id) =>
+          Buffer.from(
+            mongo.BSON.serialize({
+              _id: { _data: _id },
+              operationType: 'insert',
+              ns: { db: 'app', coll: 'documents' },
+              documentKey: { _id },
+              fullDocument: { _id }
+            })
+          )
+        ),
+        resumeToken: { _data: ids.at(-1) }
+      };
+    }
+  }
+
+  const stream = provider.openChangeStream({
+    open: () => parseMongoReplicationStream({ batches: source(), defaultSchema: 'app', multipleDatabases: false })
+  });
+  const retainedIds = [];
+  const progress = [];
+  for await (const batch of stream) {
+    for (const { event } of batch.events) {
+      if ('documentKey' in event) {
+        retainedIds.push(event.documentKey._id);
+      }
+    }
+    // Read progress only after exhausting the events, including a fully filtered batch.
+    progress.push({ token: batch.getResumeToken(), filteredCount: batch.metadata.filteredCount });
+  }
+
+  expect(retainedIds).toEqual(['public-one', 'public-two']);
+  expect(progress).toEqual([
+    { token: { _data: 'public-two' }, filteredCount: 1 },
+    { token: { _data: 'private-two' }, filteredCount: 1 }
+  ]);
+});

package/test/src/filtering_lifecycle.test.ts

@@ -0,0 +1,101 @@
+import {
+  filterMongoReplicationStream,
+  MongoReplicationFilterResult,
+  MongoReplicationQueryProvider,
+  PostImagesOption
+} from '@module/index.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { describe, expect, test } from 'vitest';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { describeWithStorage } from './util.js';
+
+const CONFIG = /* yaml */ `
+  # Sync config fixture.
+  config:
+    edition: 3
+  streams:
+    global:
+      query: SELECT _id AS id, active FROM documents
+`;
+
+/**
+ * Filter membership is decided by the provider, not by the SQL query, so bucket assertions detect leaks.
+ * Required pre/post-images let the provider distinguish entries, exits and source deletes precisely.
+ */
+const PROVIDER: MongoReplicationQueryProvider = {
+  getSnapshotFilter: () => ({ active: true }),
+  openChangeStream: ({ open }) =>
+    filterMongoReplicationStream({
+      batches: open({ imageOptions: { fullDocumentBeforeChange: 'required' } }),
+      filter: (event) => {
+        const after = 'fullDocument' in event ? event.fullDocument : undefined;
+        const before = 'fullDocumentBeforeChange' in event ? event.fullDocumentBeforeChange : undefined;
+        if (after != null && mongo.BSON.deserialize(after).active === true) {
+          return MongoReplicationFilterResult.KEEP;
+        }
+        if (before != null && mongo.BSON.deserialize(before).active === true) {
+          return MongoReplicationFilterResult.DELETE;
+        }
+        return MongoReplicationFilterResult.EXCLUDE;
+      }
+    })
+};
+
+describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB filtering lifecycle', () => {
+  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
+    test('keeps snapshot and streaming membership consistent across updates, deletes and restart', async () => {
+      const options = {
+        storageVersion,
+        mongoOptions: { postImages: PostImagesOption.READ_ONLY },
+        streamOptions: { createReplicationQueryProvider: () => PROVIDER }
+      };
+      let resumeAfter: bigint;
+      {
+        await using context = await openChangeStreamTestContext(factory, options);
+        await context.updateSyncRules(CONFIG);
+        const collection = await context.db.createCollection<{ _id: string; active: boolean }>('documents', {
+          changeStreamPreAndPostImages: { enabled: true }
+        });
+        await collection.insertMany([
+          { _id: 'one', active: true },
+          { _id: 'two', active: false },
+          { _id: 'three', active: false }
+        ]);
+        await context.replicateSnapshot();
+        let checkpoint = await context.getCheckpoint();
+        expect(await context.getBucketDataAtCheckpoint('global|0[]', checkpoint)).toMatchObject([
+          { op: 'PUT', object_id: 'one' }
+        ]);
+
+        // Enter the selected set through a real update, using the server's write-time post-image.
+        await collection.updateOne({ _id: 'two' }, { $set: { active: true } });
+        expect(await context.getBucketData('global|0[]', checkpoint.checkpoint)).toMatchObject([
+          { op: 'PUT', object_id: 'two' }
+        ]);
+        checkpoint = await context.getCheckpoint();
+
+        // Leaving the set must emit a synthetic delete rather than merely suppressing the update.
+        await collection.updateOne({ _id: 'one' }, { $set: { active: false } });
+        await collection.insertOne({ _id: 'excluded', active: false });
+        expect(await context.getBucketData('global|0[]', checkpoint.checkpoint)).toMatchObject([
+          { op: 'REMOVE', object_id: 'one' }
+        ]);
+        resumeAfter = (await context.getCheckpoint()).checkpoint;
+        await context.stop();
+
+        // Queue a source delete and another entry while replication is stopped.
+        await collection.deleteOne({ _id: 'two' });
+        await collection.updateOne({ _id: 'three' }, { $set: { active: true } });
+      }
+
+      await using resumed = await openChangeStreamTestContext(factory, { ...options, doNotClear: true });
+      await resumed.loadActiveSyncRules();
+      resumed.startStreaming();
+      expect(await resumed.getBucketData('global|0[]', resumeAfter)).toMatchObject([
+        { op: 'REMOVE', object_id: 'two' },
+        { op: 'PUT', object_id: 'three' }
+      ]);
+    });
+  });
+});

package/test/src/query_provider_availability.test.ts

@@ -0,0 +1,77 @@
+import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
+import { MetricsEngine, storage } from '@powersync/service-core';
+import { DEFAULT_HYDRATION_STATE, nodeSqlite, SqlSyncRules } from '@powersync/service-sync-rules';
+import * as sqlite from 'node:sqlite';
+import { afterEach, expect, test, vi } from 'vitest';
+import { ChangeStream } from '../../src/replication/ChangeStream.js';
+import { MongoManager } from '../../src/replication/MongoManager.js';
+import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '../../src/replication/MongoReplicationQueryProvider.js';
+import { MongoSnapshotter } from '../../src/replication/MongoSnapshotter.js';
+import { normalizeConnectionConfig } from '../../src/types/types.js';
+
+const CONFIG = /* yaml */ `
+  # Sync config fixture.
+  config:
+    edition: 3
+    source_table_options:
+      orders%:
+        mongodb_filter_expression: { $eq: ['$$doc.active', true] }
+      orders:
+        mongodb_filter_expression: disabled
+  streams:
+    orders:
+      query: SELECT * FROM orders
+`;
+const RAW_CONNECTION = { type: 'mongodb' as const, uri: 'mongodb://localhost/app' };
+const CONNECTION = { ...RAW_CONNECTION, ...normalizeConnectionConfig(RAW_CONNECTION) };
+afterEach(() => vi.restoreAllMocks());
+
+test.each([
+  { filtered: true, factory: false, blocked: true },
+  { filtered: true, factory: true, blocked: false },
+  { filtered: false, factory: false, blocked: false }
+])('checks provider availability: %j', async ({ filtered, factory, blocked }) => {
+  const yaml = filtered ? CONFIG : CONFIG.replace("{ $eq: ['$$doc.active', true] }", 'disabled');
+  const parsed = SqlSyncRules.fromYaml(yaml, { defaultSchema: 'app' });
+  const hydrated = parsed.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) });
+  const reportError = vi.fn(async (_error: unknown) => {});
+  // Stop allowed cases at the first snapshot check, without connecting to a database.
+  const checkSlot = vi
+    .spyOn(MongoSnapshotter.prototype, 'checkSlot')
+    .mockRejectedValue(new Error('snapshot check reached'));
+  const source = new MongoManager(CONNECTION);
+  const createProvider = vi.fn(() => DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER);
+  try {
+    const stream = new ChangeStream({
+      connections: source,
+      abort_signal: new AbortController().signal,
+      metrics: {} as MetricsEngine,
+      createReplicationQueryProvider: factory ? createProvider : undefined,
+      storage: {
+        replicationStreamId: 1,
+        replicationStreamName: 'availability',
+        getParsedSyncRules: () => hydrated,
+        getParsedSyncConfigSet: () => ({ syncConfigs: [parsed] }),
+        reportError,
+        logger
+      } as unknown as storage.SyncRulesBucketStorage
+    });
+    await expect(stream.replicate()).rejects.toThrow(
+      blocked ? 'no query provider factory is registered' : 'snapshot check reached'
+    );
+    if (blocked) {
+      expect(checkSlot).not.toHaveBeenCalled();
+      const error = reportError.mock.calls[0]?.[0];
+      expect(error).toBeInstanceOf(ServiceError);
+      expect(error).toMatchObject({ errorData: { code: ErrorCode.PSYNC_S1348 } });
+      expect(reportError).toHaveBeenCalledWith(
+        expect.objectContaining({ message: expect.stringContaining('deploy a new sync config') })
+      );
+    } else {
+      expect(checkSlot).toHaveBeenCalledOnce();
+    }
+    if (factory) expect(createProvider).toHaveBeenCalledWith(expect.objectContaining({ syncConfig: hydrated }));
+  } finally {
+    await source.client.close();
+  }
+});

package/test/src/query_provider_diagnostics.test.ts

@@ -0,0 +1,55 @@
+import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/index.js';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { describe, expect, test } from 'vitest';
+import { openChangeStreamTestContext } from './change_stream_test_setup.js';
+import { describeWithStorage } from './util.js';
+
+const CONFIG = /* yaml */ `
+  # Sync config fixture.
+  config:
+    edition: 3
+    source_table_options:
+      orders%:
+        mongodb_filter_expression: { $eq: ['$$doc.active', true] }
+      orders:
+        mongodb_filter_expression: disabled
+  streams:
+    orders:
+      query: SELECT _id AS id FROM orders
+`;
+
+describe('MongoDB missing-provider diagnostics', () => {
+  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
+    test('persists a missing-provider error for an active config after the provider is removed', async () => {
+      {
+        await using context = await openChangeStreamTestContext(factory, {
+          storageVersion,
+          streamOptions: { createReplicationQueryProvider: () => DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER }
+        });
+        // The concrete collection opts out, but the wildcard still requires provider support.
+        await context.updateSyncRules(CONFIG);
+        await context.db.createCollection('orders');
+        await context.replicateSnapshot();
+        await context.getCheckpoint();
+      }
+      {
+        await using context = await openChangeStreamTestContext(factory, { storageVersion, doNotClear: true });
+        await context.loadActiveSyncRules();
+        const result = await context.startStreaming();
+        expect(result.status).toBe('rejected');
+        if (result.status != 'rejected') {
+          throw new Error('Expected missing-provider rejection');
+        }
+        expect(result.reason).toBeInstanceOf(ServiceError);
+        expect(result.reason).toMatchObject({ errorData: { code: ErrorCode.PSYNC_S1348 } });
+      }
+      // Reopen storage to prove diagnostics were persisted, rather than only reported in memory.
+      await using reopened = await openChangeStreamTestContext(factory, { storageVersion, doNotClear: true });
+      const active = await reopened.factory.getActiveSyncConfig();
+      expect(active).not.toBeNull();
+      const status = await active!.content.getSyncConfigStatus();
+      expect(status?.last_fatal_error).toContain('no query provider factory is registered');
+      expect(status?.last_fatal_error).toContain('deploy a new sync config');
+    });
+  });
+});