@powersync/service-module-mongodb 0.0.0-dev-20241001150444

package/test/src/change_stream_utils.ts

@@ -0,0 +1,148 @@
+import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core';
+
+import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
+import { fromAsync } from '@core-tests/stream_utils.js';
+import { MongoManager } from '@module/replication/MongoManager.js';
+import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
+import * as mongo from 'mongodb';
+import { createCheckpoint } from '@module/replication/MongoRelation.js';
+
+/**
+ * Tests operating on the wal stream need to configure the stream and manage asynchronous
+ * replication, which gets a little tricky.
+ *
+ * This wraps a test in a function that configures all the context, and tears it down afterwards.
+ */
+export function walStreamTest(
+  factory: () => Promise<BucketStorageFactory>,
+  test: (context: ChangeStreamTestContext) => Promise<void>
+): () => Promise<void> {
+  return async () => {
+    const f = await factory();
+    const connectionManager = new MongoManager(TEST_CONNECTION_OPTIONS);
+
+    await clearTestDb(connectionManager.db);
+    const context = new ChangeStreamTestContext(f, connectionManager);
+    try {
+      await test(context);
+    } finally {
+      await context.dispose();
+    }
+  };
+}
+
+export class ChangeStreamTestContext {
+  private _walStream?: ChangeStream;
+  private abortController = new AbortController();
+  private streamPromise?: Promise<void>;
+  public storage?: SyncRulesBucketStorage;
+
+  constructor(public factory: BucketStorageFactory, public connectionManager: MongoManager) {}
+
+  async dispose() {
+    this.abortController.abort();
+    await this.streamPromise?.catch((e) => e);
+    await this.connectionManager.destroy();
+  }
+
+  get client() {
+    return this.connectionManager.client;
+  }
+
+  get db() {
+    return this.connectionManager.db;
+  }
+
+  get connectionTag() {
+    return this.connectionManager.connectionTag;
+  }
+
+  async updateSyncRules(content: string) {
+    const syncRules = await this.factory.updateSyncRules({ content: content });
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  get walStream() {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+    if (this._walStream) {
+      return this._walStream;
+    }
+    const options: ChangeStreamOptions = {
+      storage: this.storage,
+      connections: this.connectionManager,
+      abort_signal: this.abortController.signal
+    };
+    this._walStream = new ChangeStream(options);
+    return this._walStream!;
+  }
+
+  async replicateSnapshot() {
+    await this.walStream.initReplication();
+    await this.storage!.autoActivate();
+  }
+
+  startStreaming() {
+    this.streamPromise = this.walStream.streamChanges();
+  }
+
+  async getCheckpoint(options?: { timeout?: number }) {
+    let checkpoint = await Promise.race([
+      getClientCheckpoint(this.client, this.db, this.factory, { timeout: options?.timeout ?? 15_000 }),
+      this.streamPromise
+    ]);
+    if (typeof checkpoint == undefined) {
+      // This indicates an issue with the test setup - streamingPromise completed instead
+      // of getClientCheckpoint()
+      throw new Error('Test failure - streamingPromise completed');
+    }
+    return checkpoint as string;
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>(Object.entries(buckets));
+    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+  }
+
+  async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) {
+    start ??= '0';
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>([[bucket, start]]);
+    const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+    const batches = await fromAsync(batch);
+    return batches[0]?.batch.data ?? [];
+  }
+}
+
+export async function getClientCheckpoint(
+  client: mongo.MongoClient,
+  db: mongo.Db,
+  bucketStorage: BucketStorageFactory,
+  options?: { timeout?: number }
+): Promise<OpId> {
+  const start = Date.now();
+  const lsn = await createCheckpoint(client, db);
+  // This old API needs a persisted checkpoint id.
+  // Since we don't use LSNs anymore, the only way to get that is to wait.
+
+  const timeout = options?.timeout ?? 50_000;
+  let lastCp: ActiveCheckpoint | null = null;
+
+  while (Date.now() - start < timeout) {
+    const cp = await bucketStorage.getActiveCheckpoint();
+    lastCp = cp;
+    if (!cp.hasSyncRules()) {
+      throw new Error('No sync rules available');
+    }
+    if (cp.lsn && cp.lsn >= lsn) {
+      return cp.checkpoint;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 30));
+  }
+
+  throw new Error(`Timeout while waiting for checkpoint ${lsn}. Last checkpoint: ${lastCp?.lsn}`);
+}

package/test/src/env.ts

@@ -0,0 +1,7 @@
+import { utils } from '@powersync/lib-services-framework';
+
+export const env = utils.collectEnvironmentVariables({
+  MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'),
+  CI: utils.type.boolean.default('false'),
+  SLOW_TESTS: utils.type.boolean.default('false')
+});

package/test/src/mongo_test.test.ts

@@ -0,0 +1,219 @@
+import { ChangeStream } from '@module/replication/ChangeStream.js';
+import * as mongo from 'mongodb';
+import { describe, expect, test } from 'vitest';
+import { clearTestDb, connectMongoData } from './util.js';
+import { SqliteRow } from '@powersync/service-sync-rules';
+import { constructAfterRecord } from '@module/replication/MongoRelation.js';
+
+describe('mongo data types', () => {
+  async function setupTable(db: mongo.Db) {
+    await clearTestDb(db);
+  }
+
+  async function insert(collection: mongo.Collection) {
+    await collection.insertMany([
+      {
+        _id: 1 as any,
+        null: null,
+        text: 'text',
+        uuid: new mongo.UUID('baeb2514-4c57-436d-b3cc-c1256211656d'),
+        bool: true,
+        bytea: Buffer.from('test'),
+        int2: 1000,
+        int4: 1000000,
+        int8: 9007199254740993n,
+        float: 3.14
+      },
+      { _id: 2 as any, nested: { test: 'thing' } },
+      { _id: 3 as any, date: new Date('2023-03-06 15:47+02') },
+      {
+        _id: 4 as any,
+        timestamp: mongo.Timestamp.fromBits(123, 456),
+        objectId: mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')
+      }
+    ]);
+  }
+
+  async function insertNested(collection: mongo.Collection) {
+    await collection.insertMany([
+      {
+        _id: 1 as any,
+        null: [null],
+        text: ['text'],
+        uuid: [new mongo.UUID('baeb2514-4c57-436d-b3cc-c1256211656d')],
+        bool: [true],
+        bytea: [Buffer.from('test')],
+        int2: [1000],
+        int4: [1000000],
+        int8: [9007199254740993n],
+        float: [3.14]
+      },
+      { _id: 2 as any, nested: [{ test: 'thing' }] },
+      { _id: 3 as any, date: [new Date('2023-03-06 15:47+02')] },
+      {
+        _id: 10 as any,
+        timestamp: [mongo.Timestamp.fromBits(123, 456)],
+        objectId: [mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')]
+      }
+    ]);
+  }
+
+  function checkResults(transformed: Record<string, any>[]) {
+    expect(transformed[0]).toMatchObject({
+      _id: 1n,
+      text: 'text',
+      uuid: 'baeb2514-4c57-436d-b3cc-c1256211656d',
+      bool: 1n,
+      bytea: new Uint8Array([116, 101, 115, 116]),
+      int2: 1000n,
+      int4: 1000000n,
+      int8: 9007199254740993n,
+      float: 3.14,
+      null: null
+    });
+    expect(transformed[1]).toMatchObject({
+      _id: 2n,
+      nested: '{"test":"thing"}'
+    });
+
+    expect(transformed[2]).toMatchObject({
+      _id: 3n,
+      date: '2023-03-06 13:47:00.000Z'
+    });
+
+    expect(transformed[3]).toMatchObject({
+      _id: 4n,
+      objectId: '66e834cc91d805df11fa0ecb',
+      timestamp: 1958505087099n
+    });
+  }
+
+  function checkResultsNested(transformed: Record<string, any>[]) {
+    expect(transformed[0]).toMatchObject({
+      _id: 1n,
+      text: `["text"]`,
+      uuid: '["baeb2514-4c57-436d-b3cc-c1256211656d"]',
+      bool: '[1]',
+      bytea: '[null]',
+      int2: '[1000]',
+      int4: '[1000000]',
+      int8: `[9007199254740993]`,
+      float: '[3.14]',
+      null: '[null]'
+    });
+
+    // Note: Depending on to what extent we use the original postgres value, the whitespace may change, and order may change.
+    // We do expect that decimals and big numbers are preserved.
+    expect(transformed[1]).toMatchObject({
+      _id: 2n,
+      nested: '[{"test":"thing"}]'
+    });
+
+    expect(transformed[2]).toMatchObject({
+      _id: 3n,
+      date: '["2023-03-06 13:47:00.000Z"]'
+    });
+
+    expect(transformed[3]).toMatchObject({
+      _id: 10n,
+      objectId: '["66e834cc91d805df11fa0ecb"]',
+      timestamp: '[1958505087099]'
+    });
+  }
+
+  test('test direct queries', async () => {
+    const { db, client } = await connectMongoData();
+    const collection = db.collection('test_data');
+    try {
+      await setupTable(db);
+
+      await insert(collection);
+
+      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data').find().toArray())];
+
+      checkResults(transformed);
+    } finally {
+      await client.close();
+    }
+  });
+
+  test('test direct queries - arrays', async () => {
+    const { db, client } = await connectMongoData();
+    const collection = db.collection('test_data_arrays');
+    try {
+      await setupTable(db);
+
+      await insertNested(collection);
+
+      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data_arrays').find().toArray())];
+
+      checkResultsNested(transformed);
+    } finally {
+      await client.close();
+    }
+  });
+
+  test('test replication', async () => {
+    // With MongoDB, replication uses the exact same document format
+    // as normal queries. We test it anyway.
+    const { db, client } = await connectMongoData();
+    const collection = db.collection('test_data');
+    try {
+      await setupTable(db);
+
+      const stream = db.watch([], {
+        useBigInt64: true,
+        maxAwaitTimeMS: 50,
+        fullDocument: 'updateLookup'
+      });
+
+      await stream.tryNext();
+
+      await insert(collection);
+
+      const transformed = await getReplicationTx(stream, 4);
+
+      checkResults(transformed);
+    } finally {
+      await client.close();
+    }
+  });
+
+  test('test replication - arrays', async () => {
+    const { db, client } = await connectMongoData();
+    const collection = db.collection('test_data');
+    try {
+      await setupTable(db);
+
+      const stream = db.watch([], {
+        useBigInt64: true,
+        maxAwaitTimeMS: 50,
+        fullDocument: 'updateLookup'
+      });
+
+      await stream.tryNext();
+
+      await insertNested(collection);
+
+      const transformed = await getReplicationTx(stream, 4);
+
+      checkResultsNested(transformed);
+    } finally {
+      await client.close();
+    }
+  });
+});
+
+/**
+ * Return all the inserts from the first transaction in the replication stream.
+ */
+async function getReplicationTx(replicationStream: mongo.ChangeStream, count: number) {
+  let transformed: SqliteRow[] = [];
+  for await (const doc of replicationStream) {
+    transformed.push(constructAfterRecord((doc as any).fullDocument));
+    if (transformed.length == count) {
+      break;
+    }
+  }
+  return transformed;
+}

package/test/src/setup.ts

@@ -0,0 +1,7 @@
+import { container } from '@powersync/lib-services-framework';
+import { beforeAll } from 'vitest';
+
+beforeAll(() => {
+  // Executes for every test file
+  container.registerDefaults();
+});

package/test/src/util.ts

@@ -0,0 +1,52 @@
+import * as types from '@module/types/types.js';
+import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId } from '@powersync/service-core';
+
+import { env } from './env.js';
+import { logger } from '@powersync/lib-services-framework';
+import { connectMongo } from '@core-tests/util.js';
+import * as mongo from 'mongodb';
+
+// The metrics need to be initialized before they can be used
+await Metrics.initialise({
+  disable_telemetry_sharing: true,
+  powersync_instance_id: 'test',
+  internal_metrics_endpoint: 'unused.for.tests.com'
+});
+Metrics.getInstance().resetCounters();
+
+export const TEST_URI = env.MONGO_TEST_DATA_URL;
+
+export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
+  type: 'mongodb',
+  uri: TEST_URI
+});
+
+export type StorageFactory = () => Promise<BucketStorageFactory>;
+
+export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => {
+  const db = await connectMongo();
+
+  // None of the PG tests insert data into this collection, so it was never created
+  if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+    await db.db.createCollection('bucket_parameters');
+  }
+
+  await db.clear();
+
+  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
+};
+
+export async function clearTestDb(db: mongo.Db) {
+  await db.dropDatabase();
+}
+
+export async function connectMongoData() {
+  const client = new mongo.MongoClient(env.MONGO_TEST_DATA_URL, {
+    connectTimeoutMS: env.CI ? 15_000 : 5_000,
+    socketTimeoutMS: env.CI ? 15_000 : 5_000,
+    serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500,
+    useBigInt64: true
+  });
+  const dbname = new URL(env.MONGO_TEST_DATA_URL).pathname.substring(1);
+  return { client, db: client.db(dbname) };
+}

package/test/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "baseUrl": "./",
+    "noEmit": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "paths": {
+      "@/*": ["../../../packages/service-core/src/*"],
+      "@module/*": ["../src/*"],
+      "@core-tests/*": ["../../../packages/service-core/test/src/*"]
+    }
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../"
+    },
+    {
+      "path": "../../../packages/service-core/test"
+    },
+    {
+      "path": "../../../packages/service-core/"
+    }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../../packages/types"
+    },
+    {
+      "path": "../../packages/jsonbig"
+    },
+    {
+      "path": "../../packages/sync-rules"
+    },
+    {
+      "path": "../../packages/service-core"
+    },
+    {
+      "path": "../../libs/lib-services"
+    }
+  ]
+}