@powersync/service-module-mongodb 0.0.0-dev-20241111122558 → 0.0.0-dev-20241128134723

package/test/src/change_stream_utils.ts

@@ -6,30 +6,7 @@ import { MongoManager } from '@module/replication/MongoManager.js';
 import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
 import * as mongo from 'mongodb';
 import { createCheckpoint } from '@module/replication/MongoRelation.js';
-
-/**
- * Tests operating on the mongo change stream need to configure the stream and manage asynchronous
- * replication, which gets a little tricky.
- *
- * This wraps a test in a function that configures all the context, and tears it down afterwards.
- */
-export function changeStreamTest(
-  factory: () => Promise<BucketStorageFactory>,
-  test: (context: ChangeStreamTestContext) => Promise<void>
-): () => Promise<void> {
-  return async () => {
-    const f = await factory();
-    const connectionManager = new MongoManager(TEST_CONNECTION_OPTIONS);
-
-    await clearTestDb(connectionManager.db);
-    const context = new ChangeStreamTestContext(f, connectionManager);
-    try {
-      await test(context);
-    } finally {
-      await context.dispose();
-    }
-  };
-}
+import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
 
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
@@ -37,6 +14,20 @@ export class ChangeStreamTestContext {
   private streamPromise?: Promise<void>;
   public storage?: SyncRulesBucketStorage;
 
+  /**
+   * Tests operating on the mongo change stream need to configure the stream and manage asynchronous
+   * replication, which gets a little tricky.
+   *
+   * This configures all the context, and tears it down afterwards.
+   */
+  static async open(factory: () => Promise<BucketStorageFactory>, options?: Partial<NormalizedMongoConnectionConfig>) {
+    const f = await factory();
+    const connectionManager = new MongoManager({ ...TEST_CONNECTION_OPTIONS, ...options });
+
+    await clearTestDb(connectionManager.db);
+    return new ChangeStreamTestContext(f, connectionManager);
+  }
+
   constructor(
     public factory: BucketStorageFactory,
     public connectionManager: MongoManager
@@ -48,6 +39,10 @@ export class ChangeStreamTestContext {
     await this.connectionManager.destroy();
   }
 
+  async [Symbol.asyncDispose]() {
+    await this.dispose();
+  }
+
   get client() {
     return this.connectionManager.client;
   }
@@ -96,7 +91,7 @@ export class ChangeStreamTestContext {
       getClientCheckpoint(this.client, this.db, this.factory, { timeout: options?.timeout ?? 15_000 }),
       this.streamPromise
     ]);
-    if (typeof checkpoint == undefined) {
+    if (typeof checkpoint == 'undefined') {
       // This indicates an issue with the test setup - streamingPromise completed instead
       // of getClientCheckpoint()
       throw new Error('Test failure - streamingPromise completed');
@@ -110,14 +105,32 @@ export class ChangeStreamTestContext {
     return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
-  async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) {
+  async getBucketData(
+    bucket: string,
+    start?: string,
+    options?: { timeout?: number; limit?: number; chunkLimitBytes?: number }
+  ) {
     start ??= '0';
     let checkpoint = await this.getCheckpoint(options);
     const map = new Map<string, string>([[bucket, start]]);
-    const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+    const batch = this.storage!.getBucketDataBatch(checkpoint, map, {
+      limit: options?.limit,
+      chunkLimitBytes: options?.chunkLimitBytes
+    });
     const batches = await fromAsync(batch);
     return batches[0]?.batch.data ?? [];
   }
+
+  async getChecksums(buckets: string[], options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    return this.storage!.getChecksums(checkpoint, buckets);
+  }
+
+  async getChecksum(bucket: string, options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    const map = await this.storage!.getChecksums(checkpoint, [bucket]);
+    return map.get(bucket);
+  }
 }
 
 export async function getClientCheckpoint(
@@ -149,3 +162,17 @@ export async function getClientCheckpoint(
 
   throw new Error(`Timeout while waiting for checkpoint ${lsn}. Last checkpoint: ${lastCp?.lsn}`);
 }
+
+export async function setSnapshotHistorySeconds(client: mongo.MongoClient, seconds: number) {
+  const { minSnapshotHistoryWindowInSeconds: currentValue } = await client
+    .db('admin')
+    .command({ getParameter: 1, minSnapshotHistoryWindowInSeconds: 1 });
+
+  await client.db('admin').command({ setParameter: 1, minSnapshotHistoryWindowInSeconds: seconds });
+
+  return {
+    async [Symbol.asyncDispose]() {
+      await client.db('admin').command({ setParameter: 1, minSnapshotHistoryWindowInSeconds: currentValue });
+    }
+  };
+}

package/test/src/mongo_test.test.ts

@@ -1,10 +1,11 @@
 import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
 import { ChangeStream } from '@module/replication/ChangeStream.js';
 import { constructAfterRecord } from '@module/replication/MongoRelation.js';
-import { SqliteRow } from '@powersync/service-sync-rules';
+import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as mongo from 'mongodb';
 import { describe, expect, test } from 'vitest';
 import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js';
+import { PostImagesOption } from '@module/types/types.js';
 
 describe('mongo data types', () => {
   async function setupTable(db: mongo.Db) {
@@ -245,58 +246,191 @@ describe('mongo data types', () => {
   });
 
   test('connection schema', async () => {
-    const adapter = new MongoRouteAPIAdapter({
+    await using adapter = new MongoRouteAPIAdapter({
       type: 'mongodb',
       ...TEST_CONNECTION_OPTIONS
     });
-    try {
-      const db = adapter.db;
-      await clearTestDb(db);
+    const db = adapter.db;
+    await clearTestDb(db);
 
-      const collection = db.collection('test_data');
-      await setupTable(db);
-      await insert(collection);
+    const collection = db.collection('test_data');
+    await setupTable(db);
+    await insert(collection);
+
+    const schema = await adapter.getConnectionSchema();
+    const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0];
+    expect(dbSchema).not.toBeNull();
+    expect(dbSchema.tables).toMatchObject([
+      {
+        name: 'test_data',
+        columns: [
+          { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
+          { name: 'bool', sqlite_type: 4, internal_type: 'Boolean' },
+          { name: 'bytea', sqlite_type: 1, internal_type: 'Binary' },
+          { name: 'date', sqlite_type: 2, internal_type: 'Date' },
+          { name: 'decimal', sqlite_type: 2, internal_type: 'Decimal' },
+          { name: 'float', sqlite_type: 8, internal_type: 'Double' },
+          { name: 'int2', sqlite_type: 4, internal_type: 'Integer' },
+          { name: 'int4', sqlite_type: 4, internal_type: 'Integer' },
+          { name: 'int8', sqlite_type: 4, internal_type: 'Long' },
+          // We can fix these later
+          { name: 'js', sqlite_type: 2, internal_type: 'Object' },
+          { name: 'js2', sqlite_type: 2, internal_type: 'Object' },
+          { name: 'maxKey', sqlite_type: 0, internal_type: 'MaxKey' },
+          { name: 'minKey', sqlite_type: 0, internal_type: 'MinKey' },
+          { name: 'nested', sqlite_type: 2, internal_type: 'Object' },
+          { name: 'null', sqlite_type: 0, internal_type: 'Null' },
+          { name: 'objectId', sqlite_type: 2, internal_type: 'ObjectId' },
+          // We can fix these later
+          { name: 'pointer', sqlite_type: 2, internal_type: 'Object' },
+          { name: 'pointer2', sqlite_type: 2, internal_type: 'Object' },
+          { name: 'regexp', sqlite_type: 2, internal_type: 'RegExp' },
+          // Can fix this later
+          { name: 'symbol', sqlite_type: 2, internal_type: 'String' },
+          { name: 'text', sqlite_type: 2, internal_type: 'String' },
+          { name: 'timestamp', sqlite_type: 4, internal_type: 'Timestamp' },
+          { name: 'undefined', sqlite_type: 0, internal_type: 'Null' },
+          { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' }
+        ]
+      }
+    ]);
+  });
+
+  test('validate postImages', async () => {
+    await using adapter = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS,
+      postImages: PostImagesOption.READ_ONLY
+    });
+    const db = adapter.db;
+    await clearTestDb(db);
+
+    const collection = db.collection('test_data');
+    await setupTable(db);
+    await insert(collection);
+
+    const rules = SqlSyncRules.fromYaml(
+      `
+bucket_definitions:
+  global:
+    data:
+      - select _id as id, * from test_data
 
-      const schema = await adapter.getConnectionSchema();
-      const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0];
-      expect(dbSchema).not.toBeNull();
-      expect(dbSchema.tables).toMatchObject([
+      `,
+      {
+        ...adapter.getParseSyncRulesOptions(),
+        // No schema-based validation at this point
+        schema: undefined
+      }
+    );
+    const source_table_patterns = rules.getSourceTables();
+    const results = await adapter.getDebugTablesInfo(source_table_patterns, rules);
+
+    const result = results[0];
+    expect(result).not.toBeNull();
+    expect(result.table).toMatchObject({
+      schema: 'powersync_test_data',
+      name: 'test_data',
+      replication_id: ['_id'],
+      data_queries: true,
+      parameter_queries: false,
+      errors: [
         {
-          name: 'test_data',
-          columns: [
-            { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
-            { name: 'bool', sqlite_type: 4, internal_type: 'Boolean' },
-            { name: 'bytea', sqlite_type: 1, internal_type: 'Binary' },
-            { name: 'date', sqlite_type: 2, internal_type: 'Date' },
-            { name: 'decimal', sqlite_type: 2, internal_type: 'Decimal' },
-            { name: 'float', sqlite_type: 8, internal_type: 'Double' },
-            { name: 'int2', sqlite_type: 4, internal_type: 'Integer' },
-            { name: 'int4', sqlite_type: 4, internal_type: 'Integer' },
-            { name: 'int8', sqlite_type: 4, internal_type: 'Long' },
-            // We can fix these later
-            { name: 'js', sqlite_type: 2, internal_type: 'Object' },
-            { name: 'js2', sqlite_type: 2, internal_type: 'Object' },
-            { name: 'maxKey', sqlite_type: 0, internal_type: 'MaxKey' },
-            { name: 'minKey', sqlite_type: 0, internal_type: 'MinKey' },
-            { name: 'nested', sqlite_type: 2, internal_type: 'Object' },
-            { name: 'null', sqlite_type: 0, internal_type: 'Null' },
-            { name: 'objectId', sqlite_type: 2, internal_type: 'ObjectId' },
-            // We can fix these later
-            { name: 'pointer', sqlite_type: 2, internal_type: 'Object' },
-            { name: 'pointer2', sqlite_type: 2, internal_type: 'Object' },
-            { name: 'regexp', sqlite_type: 2, internal_type: 'RegExp' },
-            // Can fix this later
-            { name: 'symbol', sqlite_type: 2, internal_type: 'String' },
-            { name: 'text', sqlite_type: 2, internal_type: 'String' },
-            { name: 'timestamp', sqlite_type: 4, internal_type: 'Timestamp' },
-            { name: 'undefined', sqlite_type: 0, internal_type: 'Null' },
-            { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' }
-          ]
+          level: 'fatal',
+          message: 'changeStreamPreAndPostImages not enabled on powersync_test_data.test_data'
         }
-      ]);
-    } finally {
-      await adapter.shutdown();
-    }
+      ]
+    });
+  });
+
+  test('validate postImages - auto-configure', async () => {
+    await using adapter = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS,
+      postImages: PostImagesOption.AUTO_CONFIGURE
+    });
+    const db = adapter.db;
+    await clearTestDb(db);
+
+    const collection = db.collection('test_data');
+    await setupTable(db);
+    await insert(collection);
+
+    const rules = SqlSyncRules.fromYaml(
+      `
+bucket_definitions:
+  global:
+    data:
+      - select _id as id, * from test_data
+
+      `,
+      {
+        ...adapter.getParseSyncRulesOptions(),
+        // No schema-based validation at this point
+        schema: undefined
+      }
+    );
+    const source_table_patterns = rules.getSourceTables();
+    const results = await adapter.getDebugTablesInfo(source_table_patterns, rules);
+
+    const result = results[0];
+    expect(result).not.toBeNull();
+    expect(result.table).toMatchObject({
+      schema: 'powersync_test_data',
+      name: 'test_data',
+      replication_id: ['_id'],
+      data_queries: true,
+      parameter_queries: false,
+      errors: [
+        {
+          level: 'warning',
+          message:
+            'changeStreamPreAndPostImages not enabled on powersync_test_data.test_data, will be enabled automatically'
+        }
+      ]
+    });
+  });
+
+  test('validate postImages - off', async () => {
+    await using adapter = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS,
+      postImages: PostImagesOption.OFF
+    });
+    const db = adapter.db;
+    await clearTestDb(db);
+
+    const collection = db.collection('test_data');
+    await setupTable(db);
+    await insert(collection);
+
+    const rules = SqlSyncRules.fromYaml(
+      `
+bucket_definitions:
+  global:
+    data:
+      - select _id as id, * from test_data
+
+      `,
+      {
+        ...adapter.getParseSyncRulesOptions(),
+        // No schema-based validation at this point
+        schema: undefined
+      }
+    );
+    const source_table_patterns = rules.getSourceTables();
+    const results = await adapter.getDebugTablesInfo(source_table_patterns, rules);
+
+    const result = results[0];
+    expect(result).not.toBeNull();
+    expect(result.table).toMatchObject({
+      schema: 'powersync_test_data',
+      name: 'test_data',
+      replication_id: ['_id'],
+      data_queries: true,
+      parameter_queries: false,
+      errors: []
+    });
   });
 });
 

package/test/src/slow_tests.test.ts

@@ -0,0 +1,109 @@
+import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
+import { BucketStorageFactory } from '@powersync/service-core';
+import * as mongo from 'mongodb';
+import { setTimeout } from 'node:timers/promises';
+import { describe, expect, test } from 'vitest';
+import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js';
+import { env } from './env.js';
+
+type StorageFactory = () => Promise<BucketStorageFactory>;
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data"
+`;
+
+describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () {
+  if (env.CI || env.SLOW_TESTS) {
+    defineSlowTests(MONGO_STORAGE_FACTORY);
+  } else {
+    // Need something in this file.
+    test('no-op', () => {});
+  }
+});
+
+function defineSlowTests(factory: StorageFactory) {
+  test('replicating snapshot with lots of data', async () => {
+    await using context = await ChangeStreamTestContext.open(factory);
+    // Test with low minSnapshotHistoryWindowInSeconds, to trigger:
+    // > Read timestamp .. is older than the oldest available timestamp.
+    // This happened when we had {snapshot: true} in the initial
+    // snapshot session.
+    await using _ = await setSnapshotHistorySeconds(context.client, 1);
+    const { db } = context;
+    await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description, num FROM "test_data1"
+      - SELECT _id as id, description, num FROM "test_data2"
+      `);
+
+    const collection1 = db.collection('test_data1');
+    const collection2 = db.collection('test_data2');
+
+    let operations: mongo.AnyBulkWriteOperation[] = [];
+    for (let i = 0; i < 10_000; i++) {
+      operations.push({ insertOne: { document: { description: `pre${i}`, num: i } } });
+    }
+    await collection1.bulkWrite(operations);
+    await collection2.bulkWrite(operations);
+
+    await context.replicateSnapshot();
+    context.startStreaming();
+    const checksum = await context.getChecksum('global[]');
+    expect(checksum).toMatchObject({
+      count: 20_000
+    });
+  });
+
+  test('writes concurrently with snapshot', async () => {
+    // If there is an issue with snapshotTime (the start LSN for the
+    // changestream), we may miss updates, which this test would
+    // hopefully catch.
+
+    await using context = await ChangeStreamTestContext.open(factory);
+    const { db } = context;
+    await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description, num FROM "test_data"
+      `);
+
+    const collection = db.collection('test_data');
+
+    let operations: mongo.AnyBulkWriteOperation[] = [];
+    for (let i = 0; i < 5_000; i++) {
+      operations.push({ insertOne: { document: { description: `pre${i}`, num: i } } });
+    }
+    await collection.bulkWrite(operations);
+
+    const snapshotPromise = context.replicateSnapshot();
+
+    for (let i = 49; i >= 0; i--) {
+      await collection.updateMany(
+        { num: { $gte: i * 100, $lt: i * 100 + 100 } },
+        { $set: { description: 'updated' + i } }
+      );
+      await setTimeout(20);
+    }
+
+    await snapshotPromise;
+    context.startStreaming();
+
+    const data = await context.getBucketData('global[]', undefined, { limit: 50_000, chunkLimitBytes: 60_000_000 });
+
+    const preDocuments = data.filter((d) => JSON.parse(d.data! as string).description.startsWith('pre')).length;
+    const updatedDocuments = data.filter((d) => JSON.parse(d.data! as string).description.startsWith('updated')).length;
+
+    // If the test works properly, preDocuments should be around 2000-3000.
+    // The total should be around 9000-9900.
+    // However, it is very sensitive to timing, so we allow a wide range.
+    // updatedDocuments must be strictly >= 5000, otherwise something broke.
+    expect(updatedDocuments).toBeGreaterThanOrEqual(5_000);
+    expect(preDocuments).toBeLessThanOrEqual(5_000);
+  });
+}