@powersync/service-module-mongodb 0.0.0-dev-20241001150444

package/src/replication/MongoErrorRateLimiter.ts

@@ -0,0 +1,45 @@
+import { setTimeout } from 'timers/promises';
+import { ErrorRateLimiter } from '@powersync/service-core';
+
+export class MongoErrorRateLimiter implements ErrorRateLimiter {
+  nextAllowed: number = Date.now();
+
+  async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
+    const delay = Math.max(0, this.nextAllowed - Date.now());
+    // Minimum delay between connections, even without errors
+    this.setDelay(500);
+    await setTimeout(delay, undefined, { signal: options?.signal });
+  }
+
+  mayPing(): boolean {
+    return Date.now() >= this.nextAllowed;
+  }
+
+  reportError(e: any): void {
+    // FIXME: Check mongodb-specific requirements
+    const message = (e.message as string) ?? '';
+    if (message.includes('password authentication failed')) {
+      // Wait 15 minutes, to avoid triggering Supabase's fail2ban
+      this.setDelay(900_000);
+    } else if (message.includes('ENOTFOUND')) {
+      // DNS lookup issue - incorrect URI or deleted instance
+      this.setDelay(120_000);
+    } else if (message.includes('ECONNREFUSED')) {
+      // Could be fail2ban or similar
+      this.setDelay(120_000);
+    } else if (
+      message.includes('Unable to do postgres query on ended pool') ||
+      message.includes('Postgres unexpectedly closed connection')
+    ) {
+      // Connection timed out - ignore / immediately retry
+      // We don't explicitly set the delay to 0, since there could have been another error that
+      // we need to respect.
+    } else {
+      this.setDelay(30_000);
+    }
+  }
+
+  private setDelay(delay: number) {
+    this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
+  }
+}

package/src/replication/MongoManager.ts

@@ -0,0 +1,47 @@
+import * as mongo from 'mongodb';
+import { NormalizedMongoConnectionConfig } from '../types/types.js';
+
+export class MongoManager {
+  /**
+   * Do not use this for any transactions.
+   */
+  public readonly client: mongo.MongoClient;
+  public readonly db: mongo.Db;
+
+  constructor(public options: NormalizedMongoConnectionConfig) {
+    // The pool is lazy - no connections are opened until a query is performed.
+    this.client = new mongo.MongoClient(options.uri, {
+      auth: {
+        username: options.username,
+        password: options.password
+      },
+      // Time for connection to timeout
+      connectTimeoutMS: 5_000,
+      // Time for individual requests to timeout
+      socketTimeoutMS: 60_000,
+      // How long to wait for new primary selection
+      serverSelectionTimeoutMS: 30_000,
+
+      // Avoid too many connections:
+      // 1. It can overwhelm the source database.
+      // 2. Processing too many queries in parallel can cause the process to run out of memory.
+      maxPoolSize: 8,
+
+      maxConnecting: 3,
+      maxIdleTimeMS: 60_000
+    });
+    this.db = this.client.db(options.database, {});
+  }
+
+  public get connectionTag() {
+    return this.options.tag;
+  }
+
+  async end(): Promise<void> {
+    await this.client.close();
+  }
+
+  async destroy() {
+    // TODO: Implement?
+  }
+}

package/src/replication/MongoRelation.ts

@@ -0,0 +1,156 @@
+import { storage } from '@powersync/service-core';
+import { SqliteRow, SqliteValue, toSyncRulesRow } from '@powersync/service-sync-rules';
+import * as mongo from 'mongodb';
+import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
+
+export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor {
+  return {
+    name: source.coll,
+    schema: source.db,
+    objectId: source.coll,
+    replicationColumns: [{ name: '_id' }]
+  } satisfies storage.SourceEntityDescriptor;
+}
+
+export function getMongoLsn(timestamp: mongo.Timestamp) {
+  const a = timestamp.high.toString(16).padStart(8, '0');
+  const b = timestamp.low.toString(16).padStart(8, '0');
+  return a + b;
+}
+
+export function mongoLsnToTimestamp(lsn: string | null) {
+  if (lsn == null) {
+    return null;
+  }
+  const a = parseInt(lsn.substring(0, 8), 16);
+  const b = parseInt(lsn.substring(8, 16), 16);
+  return mongo.Timestamp.fromBits(b, a);
+}
+
+export function constructAfterRecord(document: mongo.Document): SqliteRow {
+  let record: SqliteRow = {};
+  for (let key of Object.keys(document)) {
+    record[key] = toMongoSyncRulesValue(document[key]);
+  }
+  return record;
+}
+
+export function toMongoSyncRulesValue(data: any): SqliteValue {
+  const autoBigNum = true;
+  if (data == null) {
+    // null or undefined
+    return data;
+  } else if (typeof data == 'string') {
+    return data;
+  } else if (typeof data == 'number') {
+    if (Number.isInteger(data) && autoBigNum) {
+      return BigInt(data);
+    } else {
+      return data;
+    }
+  } else if (typeof data == 'bigint') {
+    return data;
+  } else if (typeof data == 'boolean') {
+    return data ? 1n : 0n;
+  } else if (data instanceof mongo.ObjectId) {
+    return data.toHexString();
+  } else if (data instanceof mongo.UUID) {
+    return data.toHexString();
+  } else if (data instanceof Date) {
+    return data.toISOString().replace('T', ' ');
+  } else if (data instanceof mongo.Binary) {
+    return new Uint8Array(data.buffer);
+  } else if (data instanceof mongo.Long) {
+    return data.toBigInt();
+  } else if (Array.isArray(data)) {
+    // We may be able to avoid some parse + stringify cycles here for JsonSqliteContainer.
+    return JSONBig.stringify(data.map((element) => filterJsonData(element)));
+  } else if (data instanceof Uint8Array) {
+    return data;
+  } else if (data instanceof JsonContainer) {
+    return data.toString();
+  } else if (typeof data == 'object') {
+    let record: Record<string, any> = {};
+    for (let key of Object.keys(data)) {
+      record[key] = filterJsonData(data[key]);
+    }
+    return JSONBig.stringify(record);
+  } else {
+    return null;
+  }
+}
+
+const DEPTH_LIMIT = 20;
+
+function filterJsonData(data: any, depth = 0): any {
+  const autoBigNum = true;
+  if (depth > DEPTH_LIMIT) {
+    // This is primarily to prevent infinite recursion
+    throw new Error(`json nested object depth exceeds the limit of ${DEPTH_LIMIT}`);
+  }
+  if (data == null) {
+    return data; // null or undefined
+  } else if (typeof data == 'string') {
+    return data;
+  } else if (typeof data == 'number') {
+    if (autoBigNum && Number.isInteger(data)) {
+      return BigInt(data);
+    } else {
+      return data;
+    }
+  } else if (typeof data == 'boolean') {
+    return data ? 1n : 0n;
+  } else if (typeof data == 'bigint') {
+    return data;
+  } else if (data instanceof Date) {
+    return data.toISOString().replace('T', ' ');
+  } else if (data instanceof mongo.ObjectId) {
+    return data.toHexString();
+  } else if (data instanceof mongo.UUID) {
+    return data.toHexString();
+  } else if (data instanceof mongo.Binary) {
+    return undefined;
+  } else if (data instanceof mongo.Long) {
+    return data.toBigInt();
+  } else if (Array.isArray(data)) {
+    return data.map((element) => filterJsonData(element, depth + 1));
+  } else if (ArrayBuffer.isView(data)) {
+    return undefined;
+  } else if (data instanceof JsonContainer) {
+    // Can be stringified directly when using our JSONBig implementation
+    return data;
+  } else if (typeof data == 'object') {
+    let record: Record<string, any> = {};
+    for (let key of Object.keys(data)) {
+      record[key] = filterJsonData(data[key], depth + 1);
+    }
+    return record;
+  } else {
+    return undefined;
+  }
+}
+
+export async function createCheckpoint(client: mongo.MongoClient, db: mongo.Db): Promise<string> {
+  const session = client.startSession();
+  try {
+    const result = await db.collection('_powersync_checkpoints').findOneAndUpdate(
+      {
+        _id: 'checkpoint' as any
+      },
+      {
+        $inc: { i: 1 }
+      },
+      {
+        upsert: true,
+        returnDocument: 'after',
+        session
+      }
+    );
+    const time = session.operationTime!;
+    // console.log('marked checkpoint at', time, getMongoLsn(time));
+    // TODO: Use the above when we support custom write checkpoints
+    return getMongoLsn(time);
+  } finally {
+    await session.endSession();
+  }
+}

package/src/replication/replication-index.ts

@@ -0,0 +1,4 @@
+export * from './MongoRelation.js';
+export * from './ChangeStream.js';
+export * from './ChangeStreamReplicator.js';
+export * from './ChangeStreamReplicationJob.js';

package/src/types/types.ts

@@ -0,0 +1,65 @@
+import { normalizeMongoConfig } from '@powersync/service-core';
+import * as service_types from '@powersync/service-types';
+import * as t from 'ts-codec';
+
+export const MONGO_CONNECTION_TYPE = 'mongodb' as const;
+
+export interface NormalizedMongoConnectionConfig {
+  id: string;
+  tag: string;
+
+  uri: string;
+  database: string;
+
+  username?: string;
+  password?: string;
+}
+
+export const MongoConnectionConfig = service_types.configFile.dataSourceConfig.and(
+  t.object({
+    type: t.literal(MONGO_CONNECTION_TYPE),
+    /** Unique identifier for the connection - optional when a single connection is present. */
+    id: t.string.optional(),
+    /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */
+    tag: t.string.optional(),
+    uri: t.string,
+    username: t.string.optional(),
+    password: t.string.optional(),
+    database: t.string.optional()
+  })
+);
+
+/**
+ * Config input specified when starting services
+ */
+export type MongoConnectionConfig = t.Decoded<typeof MongoConnectionConfig>;
+
+/**
+ * Resolved version of {@link MongoConnectionConfig}
+ */
+export type ResolvedConnectionConfig = MongoConnectionConfig & NormalizedMongoConnectionConfig;
+
+/**
+ * Validate and normalize connection options.
+ *
+ * Returns destructured options.
+ */
+export function normalizeConnectionConfig(options: MongoConnectionConfig): NormalizedMongoConnectionConfig {
+  const base = normalizeMongoConfig(options);
+
+  return {
+    id: options.id ?? 'default',
+    tag: options.tag ?? 'default',
+
+    ...base
+  };
+}
+
+/**
+ * Construct a mongodb URI, without username, password or ssl options.
+ *
+ * Only contains hostname, port, database.
+ */
+export function baseUri(options: NormalizedMongoConnectionConfig) {
+  return options.uri;
+}

package/test/src/change_stream.test.ts

@@ -0,0 +1,306 @@
+import { putOp, removeOp } from '@core-tests/stream_utils.js';
+import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
+import { BucketStorageFactory } from '@powersync/service-core';
+import * as crypto from 'crypto';
+import { describe, expect, test } from 'vitest';
+import { walStreamTest } from './change_stream_utils.js';
+import * as mongo from 'mongodb';
+import { setTimeout } from 'node:timers/promises';
+
+type StorageFactory = () => Promise<BucketStorageFactory>;
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data"
+`;
+
+describe(
+  'change stream - mongodb',
+  function () {
+    defineWalStreamTests(MONGO_STORAGE_FACTORY);
+  },
+  { timeout: 20_000 }
+);
+
+function defineWalStreamTests(factory: StorageFactory) {
+  test(
+    'replicating basic values',
+    walStreamTest(factory, async (context) => {
+      const { db } = context;
+      await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description, num FROM "test_data"`);
+
+      db.createCollection('test_data', {
+        changeStreamPreAndPostImages: { enabled: true }
+      });
+      const collection = db.collection('test_data');
+
+      await context.replicateSnapshot();
+
+      context.startStreaming();
+
+      const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
+      const test_id = result.insertedId;
+      await setTimeout(10);
+      await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
+      await setTimeout(10);
+      await collection.replaceOne({ _id: test_id }, { description: 'test3' });
+      await setTimeout(10);
+      await collection.deleteOne({ _id: test_id });
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
+        putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }),
+        putOp('test_data', { id: test_id.toHexString(), description: 'test3' }),
+        removeOp('test_data', test_id.toHexString())
+      ]);
+    })
+  );
+
+  test(
+    'no fullDocument available',
+    walStreamTest(factory, async (context) => {
+      const { db, client } = context;
+      await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description, num FROM "test_data"`);
+
+      db.createCollection('test_data', {
+        changeStreamPreAndPostImages: { enabled: false }
+      });
+      const collection = db.collection('test_data');
+
+      await context.replicateSnapshot();
+
+      context.startStreaming();
+
+      const session = client.startSession();
+      let test_id: mongo.ObjectId | undefined;
+      try {
+        await session.withTransaction(async () => {
+          const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n }, { session });
+          test_id = result.insertedId;
+          await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } }, { session });
+          await collection.replaceOne({ _id: test_id }, { description: 'test3' }, { session });
+          await collection.deleteOne({ _id: test_id }, { session });
+        });
+      } finally {
+        await session.endSession();
+      }
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+        // fullDocument is not available at the point this is replicated, resulting in it treated as a remove
+        removeOp('test_data', test_id!.toHexString()),
+        putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+        removeOp('test_data', test_id!.toHexString())
+      ]);
+    })
+  );
+
+  test(
+    'replicating case sensitive table',
+    walStreamTest(factory, async (context) => {
+      const { db } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT _id as id, description FROM "test_DATA"
+      `);
+
+      await context.replicateSnapshot();
+
+      context.startStreaming();
+
+      const collection = db.collection('test_DATA');
+      const result = await collection.insertOne({ description: 'test1' });
+      const test_id = result.insertedId.toHexString();
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
+    })
+  );
+
+  test(
+    'replicating large values',
+    walStreamTest(factory, async (context) => {
+      const { db } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT _id as id, name, description FROM "test_data"
+      `);
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const largeDescription = crypto.randomBytes(20_000).toString('hex');
+
+      const collection = db.collection('test_data');
+      const result = await collection.insertOne({ name: 'test1', description: largeDescription });
+      const test_id = result.insertedId;
+
+      await collection.updateOne({ _id: test_id }, { $set: { name: 'test2' } });
+
+      const data = await context.getBucketData('global[]');
+      expect(data.slice(0, 1)).toMatchObject([
+        putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription })
+      ]);
+      expect(data.slice(1)).toMatchObject([
+        putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription })
+      ]);
+    })
+  );
+
+  test(
+    'replicating dropCollection',
+    walStreamTest(factory, async (context) => {
+      const { db } = context;
+      const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data"
+  by_test_data:
+    parameters: SELECT _id as id FROM test_data WHERE id = token_parameters.user_id
+    data: []
+`;
+      await context.updateSyncRules(syncRuleContent);
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const collection = db.collection('test_data');
+      const result = await collection.insertOne({ description: 'test1' });
+      const test_id = result.insertedId.toHexString();
+
+      await collection.drop();
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data', { id: test_id, description: 'test1' }),
+        removeOp('test_data', test_id)
+      ]);
+    })
+  );
+
+  test(
+    'replicating renameCollection',
+    walStreamTest(factory, async (context) => {
+      const { db } = context;
+      const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data1"
+      - SELECT _id as id, description FROM "test_data2"
+`;
+      await context.updateSyncRules(syncRuleContent);
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      console.log('insert1', db.databaseName);
+      const collection = db.collection('test_data1');
+      const result = await collection.insertOne({ description: 'test1' });
+      const test_id = result.insertedId.toHexString();
+
+      await collection.rename('test_data2');
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data1', { id: test_id, description: 'test1' }),
+        removeOp('test_data1', test_id),
+        putOp('test_data2', { id: test_id, description: 'test1' })
+      ]);
+    })
+  );
+
+  test(
+    'initial sync',
+    walStreamTest(factory, async (context) => {
+      const { db } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      const collection = db.collection('test_data');
+      const result = await collection.insertOne({ description: 'test1' });
+      const test_id = result.insertedId.toHexString();
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
+    })
+  );
+
+  // Not correctly implemented yet
+  test.skip(
+    'large record',
+    walStreamTest(factory, async (context) => {
+      await context.updateSyncRules(`bucket_definitions:
+      global:
+        data:
+          - SELECT _id as id, description, other FROM "test_data"`);
+      const { db } = context;
+
+      await context.replicateSnapshot();
+
+      // 16MB
+      const largeDescription = crypto.randomBytes(8_000_000 - 100).toString('hex');
+
+      const collection = db.collection('test_data');
+      const result = await collection.insertOne({ description: largeDescription });
+      const test_id = result.insertedId;
+
+      await collection.updateOne({ _id: test_id }, { $set: { name: 't2' } });
+      context.startStreaming();
+
+      const data = await context.getBucketData('global[]');
+      expect(data.length).toEqual(2);
+      const row = JSON.parse(data[0].data as string);
+      delete row.description;
+      expect(row).toEqual({ id: test_id.toHexString() });
+      delete data[0].data;
+      expect(data[0]).toMatchObject({
+        object_id: test_id.toHexString(),
+        object_type: 'test_data',
+        op: 'PUT',
+        op_id: '1'
+      });
+    })
+  );
+
+  test(
+    'table not in sync rules',
+    walStreamTest(factory, async (context) => {
+      const { db } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await context.replicateSnapshot();
+
+      context.startStreaming();
+
+      const collection = db.collection('test_donotsync');
+      const result = await collection.insertOne({ description: 'test' });
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([]);
+    })
+  );
+}