@powersync/service-module-mongodb 0.15.3 → 0.16.0

package/test/src/raw_change_stream.test.ts

@@ -0,0 +1,547 @@
+import { describe, expect, test } from 'vitest';
+
+import { ChangeStreamBatch, namespaceCollection, rawChangeStream } from '@module/replication/RawChangeStream.js';
+import { getCursorBatchBytes } from '@module/replication/replication-index.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { bson } from '@powersync/service-core';
+import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
+
+describe('internal mongodb utils', () => {
+  // The implementation relies on internal APIs, so we verify this works as expected for various types of change streams.
+  test('collection change stream size tracking', async () => {
+    await testChangeStreamBsonBytes('collection');
+  });
+
+  test('db change stream size tracking', async () => {
+    await testChangeStreamBsonBytes('db');
+  });
+
+  test('cluster change stream size tracking', async () => {
+    await testChangeStreamBsonBytes('cluster');
+  });
+
+  test('cursor batch size tracking', async () => {
+    const { db, client } = await connectMongoData();
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+    await collection.insertMany([{ test: 1 }, { test: 2 }, { test: 3 }, { test: 4 }, { test: 5 }]);
+
+    const cursor = collection.find({}, { batchSize: 2 });
+    let batchBytes: number[] = [];
+    let totalBytes = 0;
+    // We use this in the same way as ChunkedSnapshotQuery
+    while (await cursor.hasNext()) {
+      batchBytes.push(getCursorBatchBytes(cursor));
+      totalBytes += batchBytes[batchBytes.length - 1];
+      cursor.readBufferedDocuments();
+    }
+
+    // 3 batches: [2, 2, 1] documents. Should not change
+    expect(batchBytes.length).toEqual(3);
+    // Current tests show 839, but this may change depending on the MongoDB version and other conditions.
+    expect(totalBytes).toBeGreaterThan(400);
+    expect(totalBytes).toBeLessThan(1200);
+  });
+
+  test('uses separate aggregate and getMore command options', async () => {
+    const { db, client } = await connectMongoData({ monitorCommands: true });
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+
+    const started: any[] = [];
+    client.on('commandStarted', (event) => {
+      if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
+        started.push(event);
+      }
+    });
+
+    const stream = rawChangeStream(
+      db,
+      [
+        {
+          $changeStream: {
+            fullDocument: 'updateLookup'
+          }
+        }
+      ],
+      {
+        batchSize: 10,
+        maxAwaitTimeMS: 50,
+        maxTimeMS: 1_000
+      }
+    );
+
+    await stream.next();
+    await collection.insertOne({ test: 1 });
+    const nextBatch = await readUntilNonEmptyBatch(stream);
+    await stream.return?.();
+
+    expect(nextBatch.events).toHaveLength(1);
+
+    const aggregate = started.find((event) => event.commandName == 'aggregate');
+    const getMore = started.find((event) => event.commandName == 'getMore');
+
+    expect(aggregate?.command.cursor?.batchSize).toEqual(1);
+    expect(aggregate?.command.maxTimeMS).toEqual(1_000);
+    expect(getMore?.command.batchSize).toEqual(10);
+    expect(getMore?.command.maxTimeMS).toEqual(50);
+  });
+
+  test('should resume on MaxTimeMSExpired from getMore', async (ctx) => {
+    const { db, client } = await connectMongoData({ monitorCommands: true });
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await using failCommand = await requireFailCommand(client, ctx);
+
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+
+    const started: any[] = [];
+    client.on('commandStarted', (event) => {
+      if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
+        started.push(event);
+      }
+    });
+
+    const stream = rawChangeStream(
+      db,
+      [
+        {
+          $changeStream: {
+            fullDocument: 'updateLookup'
+          }
+        }
+      ],
+      {
+        batchSize: 3,
+        maxAwaitTimeMS: 50,
+        maxTimeMS: 1_000
+        // To get more details when debugging, enable the logger:
+        // logger: logger
+      }
+    );
+
+    await stream.next();
+
+    await failCommand.configure({
+      mode: { times: 1 },
+      data: {
+        failCommands: ['getMore'],
+        errorCode: 50 // MaxTimeMSExpired
+      }
+    });
+
+    for (let i = 1; i <= 8; i++) {
+      await collection.insertOne({ test: i });
+    }
+
+    // Test the exponentially-increasing batch size after the retry
+    {
+      // This will fail the getMore, then retry with aggregate with batchSize 1
+      const batch = await readUntilNonEmptyBatch(stream, 10);
+      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+        { test: 1 }
+      ]);
+    }
+    {
+      // This will be a getMore with batchSize 2
+      const batch = await readUntilNonEmptyBatch(stream, 10);
+      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+        { test: 2 },
+        { test: 3 }
+      ]);
+    }
+    {
+      // At this point, this batch size is at the original size of 3 again.
+      const batch = await readUntilNonEmptyBatch(stream, 10);
+      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+        { test: 4 },
+        { test: 5 },
+        { test: 6 }
+      ]);
+    }
+    {
+      const batch = await readUntilNonEmptyBatch(stream, 10);
+      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+        { test: 7 },
+        { test: 8 }
+      ]);
+    }
+
+    const aggregateCommands = started.filter((event) => event.commandName == 'aggregate');
+    expect(aggregateCommands.length).toBeGreaterThanOrEqual(2);
+    expect(aggregateCommands[0].command.pipeline).toEqual([
+      {
+        $changeStream: {
+          fullDocument: 'updateLookup'
+        }
+      }
+    ]);
+    expect(aggregateCommands[1].command.pipeline[0]?.$changeStream?.resumeAfter).toBeDefined();
+
+    await stream?.return().catch(() => {});
+  });
+
+  async function testChangeStreamBsonBytes(type: 'db' | 'collection' | 'cluster') {
+    const { db, client } = await connectMongoData();
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+
+    let stream: AsyncIterableIterator<ChangeStreamBatch>;
+    const pipeline = [
+      {
+        $changeStream: {
+          fullDocument: 'updateLookup',
+          allChangesForCluster: type == 'cluster'
+        }
+      }
+    ];
+    if (type === 'collection') {
+      stream = rawChangeStream(db, pipeline, {
+        batchSize: 10,
+        maxAwaitTimeMS: 5,
+        maxTimeMS: 1_000,
+        collection: collection.collectionName
+      });
+    } else if (type === 'db') {
+      stream = rawChangeStream(db, pipeline, {
+        batchSize: 10,
+        maxAwaitTimeMS: 5,
+        maxTimeMS: 1_000
+      });
+    } else {
+      stream = rawChangeStream(client.db('admin'), pipeline, {
+        batchSize: 10,
+        maxAwaitTimeMS: 5,
+        maxTimeMS: 1_000
+      });
+    }
+
+    let batchBytes: number[] = [];
+    let totalBytes = 0;
+
+    const readAll = async () => {
+      while (true) {
+        const next = await stream.next();
+        if (next.done) {
+          break;
+        }
+        const bytes = next.value.byteSize;
+        batchBytes.push(bytes);
+        totalBytes += bytes;
+
+        if (next.value.events.length == 0) {
+          break;
+        }
+      }
+    };
+
+    await readAll();
+
+    await collection.insertOne({ test: 1 });
+    await readAll();
+    await collection.insertOne({ test: 2 });
+    await readAll();
+    await collection.insertOne({ test: 3 });
+    await readAll();
+
+    await stream.return?.();
+
+    // The exact length by vary based on exact batching logic, but we do want to know when it changes.
+    // Note: If this causes unstable tests, we can relax this check.
+    expect(batchBytes.length).toEqual(7);
+
+    // Current tests show 4464-4576 bytes for the size, depending on the type of change stream.
+    // This can easily vary based on the mongodb version and general conditions, so we just check the general range.
+    // For the most part, if any bytes are reported, the tracking is working.
+    expect(totalBytes).toBeGreaterThan(2000);
+    expect(totalBytes).toBeLessThan(8000);
+  }
+
+  test('should resume on missing cursor (1)', async () => {
+    // Many resumable errors are difficult to simulate, but CursorNotFound is easy.
+
+    const { db, client } = await connectMongoData();
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+
+    const pipeline = [
+      {
+        $changeStream: {
+          fullDocument: 'updateLookup'
+        }
+      }
+    ];
+    const stream = rawChangeStream(db, pipeline, {
+      batchSize: 10,
+      maxAwaitTimeMS: 5,
+      maxTimeMS: 1_000
+    });
+
+    let readDocs: any[] = [];
+    const readAll = async () => {
+      while (true) {
+        const next = await stream.next();
+        if (next.done) {
+          break;
+        }
+
+        if (next.value.events.length == 0) {
+          break;
+        }
+
+        readDocs.push(...next.value.events.map((e) => bson.deserialize(e, { useBigInt64: true })));
+      }
+    };
+
+    await readAll();
+
+    await collection.insertOne({ test: 1 });
+    await readAll();
+    await collection.insertOne({ test: 2 });
+    await readAll();
+    await collection.insertOne({ test: 3 });
+    await killChangeStreamCursor(db, client);
+    await collection.insertOne({ test: 4 });
+    await readAll();
+
+    await stream.return?.();
+
+    expect(readDocs.map((doc) => doc.fullDocument)).toMatchObject([{ test: 1 }, { test: 2 }, { test: 3 }, { test: 4 }]);
+  });
+
+  test('should resume on missing cursor (2)', async () => {
+    const { db, client } = await connectMongoData();
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+
+    const currentOpStream = rawChangeStream(
+      db,
+      [
+        {
+          $changeStream: {
+            fullDocument: 'updateLookup'
+          }
+        }
+      ],
+      {
+        batchSize: 10,
+        maxAwaitTimeMS: 5,
+        maxTimeMS: 1_000
+      }
+    );
+    const firstBatch = await currentOpStream.next();
+    await currentOpStream.return();
+    const resumeAfter = firstBatch.value!.resumeToken;
+
+    const stream = rawChangeStream(
+      db,
+      [
+        {
+          $changeStream: {
+            fullDocument: 'updateLookup',
+            resumeAfter
+          }
+        }
+      ],
+      {
+        batchSize: 10,
+        maxAwaitTimeMS: 5,
+        maxTimeMS: 1_000
+      }
+    );
+
+    let readDocs: any[] = [];
+    const readAll = async () => {
+      while (true) {
+        const next = await stream.next();
+        if (next.done) {
+          break;
+        }
+
+        if (next.value.events.length == 0) {
+          break;
+        }
+
+        readDocs.push(...next.value.events.map((e) => bson.deserialize(e, { useBigInt64: true })));
+      }
+    };
+
+    await readAll();
+
+    await collection.insertOne({ test: 1 });
+    await readAll();
+    await collection.insertOne({ test: 2 });
+    await readAll();
+    await collection.insertOne({ test: 3 });
+    await killChangeStreamCursor(db, client);
+    await collection.insertOne({ test: 4 });
+    await readAll();
+
+    await stream.return?.();
+
+    expect(readDocs.map((doc) => doc.fullDocument)).toMatchObject([{ test: 1 }, { test: 2 }, { test: 3 }, { test: 4 }]);
+  });
+
+  test('should cleanly abort a stream between events', async () => {
+    const { db, client } = await connectMongoData();
+    const abortController = new AbortController();
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+
+    const pipeline = [
+      {
+        $changeStream: {
+          fullDocument: 'updateLookup'
+        }
+      }
+    ];
+    const stream = rawChangeStream(db, pipeline, {
+      batchSize: 10,
+      maxAwaitTimeMS: 5,
+      maxTimeMS: 1_000,
+      signal: abortController.signal
+    });
+
+    let readDocs: any[] = [];
+    const readAll = async () => {
+      while (true) {
+        const next = await stream.next();
+        if (next.done) {
+          break;
+        }
+
+        if (next.value.events.length == 0) {
+          break;
+        }
+
+        readDocs.push(...next.value.events.map((e) => bson.deserialize(e, { useBigInt64: true })));
+      }
+    };
+
+    await readAll();
+
+    await collection.insertOne({ test: 1 });
+    await readAll();
+    await collection.insertOne({ test: 2 });
+    await readAll();
+    abortController.abort(new Error('test abort'));
+    await collection.insertOne({ test: 3 });
+    await expect(readAll()).rejects.toMatchObject({ message: 'test abort' });
+
+    expect(readDocs.map((doc) => doc.fullDocument)).toMatchObject([{ test: 1 }, { test: 2 }]);
+  });
+
+  test('should cleanly abort a stream in an event', async () => {
+    const { db, client } = await connectMongoData();
+    const abortController = new AbortController();
+    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+    await clearTestDb(db);
+    const collection = db.collection('test_data');
+
+    const pipeline = [
+      {
+        $changeStream: {
+          fullDocument: 'updateLookup'
+        }
+      }
+    ];
+    const stream = rawChangeStream(db, pipeline, {
+      batchSize: 10,
+      maxAwaitTimeMS: 200,
+      maxTimeMS: 1_000,
+      signal: abortController.signal
+    });
+
+    let readDocs: any[] = [];
+    const readAll = async () => {
+      while (true) {
+        const next = await stream.next();
+        if (next.done) {
+          break;
+        }
+
+        if (next.value.events.length == 0) {
+          break;
+        }
+
+        readDocs.push(...next.value.events.map((e) => bson.deserialize(e, { useBigInt64: true })));
+      }
+    };
+
+    await readAll();
+
+    await collection.insertOne({ test: 1 });
+    await readAll();
+    // This is specifically a readAll() without an insert in between, to trigger the longer await
+    // period.
+    let readPromise = readAll();
+    abortController.abort(new Error('test abort'));
+    await expect(readPromise).rejects.toMatchObject({ message: 'test abort' });
+
+    expect(readDocs.map((doc) => doc.fullDocument)).toMatchObject([{ test: 1 }]);
+  });
+});
+
+async function killChangeStreamCursor(db: mongo.Db, client: mongo.MongoClient) {
+  const ops = await client
+    .db('admin')
+    .aggregate<CurrentOpIdleCursor>([{ $currentOp: { idleCursors: true } }, { $match: { type: 'idleCursor' } }])
+    .toArray();
+
+  const ns = `${db.databaseName}.$cmd.aggregate`;
+  const op = ops.find((op) => {
+    const command = op.cursor?.originatingCommand;
+    return op.ns == ns && Array.isArray(command?.pipeline) && command.pipeline[0]?.$changeStream != null;
+  });
+
+  if (op?.cursor == null) {
+    throw new Error(
+      `Could not find change stream cursor. Idle cursors: ${JSON.stringify(
+        ops.map((op) => ({
+          ns: op.ns,
+          type: op.type,
+          cursorId: op.cursor?.cursorId?.toString(),
+          aggregate: op.cursor?.originatingCommand?.aggregate,
+          pipeline: op.cursor?.originatingCommand?.pipeline
+        }))
+      )}`
+    );
+  }
+
+  await db.command({
+    killCursors: namespaceCollection(op.ns),
+    cursors: [op.cursor.cursorId]
+  });
+}
+
+type CurrentOpIdleCursor = {
+  ns: string;
+  type: string;
+  cursor?: {
+    cursorId: bigint;
+    originatingCommand?: {
+      aggregate?: unknown;
+      pipeline?: any[];
+    };
+  };
+};
+
+async function readUntilNonEmptyBatch(stream: AsyncIterableIterator<ChangeStreamBatch>, maxEmptyBatches: number = 5) {
+  for (let i = 0; i < maxEmptyBatches; i++) {
+    const next = await stream.next();
+    if (next.done) {
+      throw new Error('Change stream ended unexpectedly');
+    }
+    if (next.value.events.length > 0) {
+      return next.value;
+    }
+  }
+
+  throw new Error(`Did not receive a non-empty batch after ${maxEmptyBatches} empty batches`);
+}

package/test/src/resume.test.ts

@@ -15,7 +15,17 @@ function defineResumeTest({ factory: factoryGenerator, storageVersion }: Storage
     return ChangeStreamTestContext.open(factoryGenerator, { ...options, storageVersion });
   };
 
-  test('resuming with a different source database', async () => {
+  test.skip('resuming with a different source database', async () => {
+    // This test is not functioning anymore.
+    // Previously, we mostly used individual change stream _id's for resumeTokens. Those would become invalid
+    // when the database is changed.
+    // Now, we mostly use postBatchResumeToken when we can, which do not become invalidated in this case.
+    // This is recommended by the change stream spec.
+    // The old behavior wasn't 100% consistent either - it _could_ use postBatchResumeToken, in which
+    // case it would similarly not be invalidated.
+    // We can revisit the logic to invalidate the change stream at a later point, potentially by
+    // keeping track of the source database name.
+
     await using context = await openContext();
     const { db } = context;
 
@@ -55,7 +65,7 @@ function defineResumeTest({ factory: factoryGenerator, storageVersion }: Storage
     );
     const factory = await factoryGenerator({ doNotClear: true });
 
-    // Create a new context without updating the sync rules
+    // Create a new context without updating the sync config
     await using context2 = new ChangeStreamTestContext(factory, connectionManager, {}, storageVersion);
     const activeContent = await factory.getActiveSyncRulesContent();
     context2.storage = factory.getInstance(activeContent!);

package/test/src/util.ts

@@ -9,7 +9,7 @@ import {
   TestStorageConfig,
   TestStorageFactory
 } from '@powersync/service-core';
-import { describe, TestOptions } from 'vitest';
+import { describe, TestContext, TestOptions } from 'vitest';
 import { env } from './env.js';
 
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
@@ -62,13 +62,66 @@ export async function clearTestDb(db: mongo.Db) {
   await db.dropDatabase();
 }
 
-export async function connectMongoData() {
+export async function connectMongoData(options: mongo.MongoClientOptions = {}) {
   const client = new mongo.MongoClient(env.MONGO_TEST_DATA_URL, {
     connectTimeoutMS: env.CI ? 15_000 : 5_000,
     socketTimeoutMS: env.CI ? 15_000 : 5_000,
     serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500,
-    ...BSON_DESERIALIZE_DATA_OPTIONS
+    ...BSON_DESERIALIZE_DATA_OPTIONS,
+    ...options
   });
   const dbname = new URL(env.MONGO_TEST_DATA_URL).pathname.substring(1);
   return { client, db: client.db(dbname) };
 }
+
+/**
+ * This allows us to inject custom failures into commands on the mongodb server.
+ *
+ * For this to work, mongodb must be started with `--setParameter enableTestCommands=1`.
+ *
+ * We require this in CI, but in local development we skip the test if it's not configured.
+ *
+ * https://github.com/mongodb/mongo/wiki/The-failCommand-fail-point
+ */
+export async function requireFailCommand(client: mongo.MongoClient, ctx: TestContext) {
+  try {
+    await client.db('admin').command({ configureFailPoint: 'failCommand', mode: 'off' });
+  } catch (e: any) {
+    const codeName = e?.codeName;
+    const message = e?.message ?? String(e);
+
+    if (
+      codeName == 'CommandNotFound' ||
+      codeName == 'Unauthorized' ||
+      message.includes('no such command') ||
+      message.includes('enableTestCommands')
+    ) {
+      if (process.env.CI) {
+        // In CI we want to fail if failCommand is not supported, as that likely means something is wrong with the test environment setup.
+        throw e;
+      }
+      // In local development, we skip the test if failCommand is not supported, as developers may be running against a variety of MongoDB versions and configurations.
+      ctx.skip(`failCommand not supported: ${codeName ?? message}`);
+    }
+
+    throw e;
+  }
+
+  return {
+    async configure(data: Omit<mongo.Document, 'configureFailPoint'>) {
+      await client.db('admin').command({
+        configureFailPoint: 'failCommand',
+        ...data
+      });
+    },
+    async [Symbol.asyncDispose]() {
+      await client
+        .db('admin')
+        .command({
+          configureFailPoint: 'failCommand',
+          mode: 'off'
+        })
+        .catch(() => {});
+    }
+  };
+}

package/test/tsconfig.json

@@ -1,7 +1,6 @@
 {
   "extends": "../../../tsconfig.tests.json",
   "compilerOptions": {
-    "baseUrl": "./",
     "paths": {
       "@/*": ["../../../packages/service-core/src/*"],
       "@module/*": ["../src/*"],

package/tsconfig.scripts.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "target": "esnext",
+    "noEmit": true,
+    "composite": false,
+    "types": ["node"],
+    "skipLibCheck": true
+  },
+  "include": ["scripts/**/*.mts"]
+}