@powersync/service-module-mongodb 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219110735

package/src/locks/MonogLocks.ts

@@ -0,0 +1,147 @@
+import * as framework from '@powersync/lib-services-framework';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+
+/**
+ * Lock Document Schema
+ */
+export type Lock = {
+  name: string;
+  active_lock?: {
+    lock_id: bson.ObjectId;
+    ts: Date;
+  };
+};
+
+export type Collection = mongo.Collection<Lock>;
+
+export type AcquireLockParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL of the lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+const DEFAULT_LOCK_TIMEOUT = 60 * 1000; // 1 minute
+
+const acquireLock = async (collection: Collection, params: AcquireLockParams) => {
+  const now = new Date();
+  const lock_timeout = params.timeout ?? DEFAULT_LOCK_TIMEOUT;
+  const lock_id = new bson.ObjectId();
+
+  await collection.updateOne(
+    {
+      name: params.name
+    },
+    {
+      $setOnInsert: {
+        name: params.name
+      }
+    },
+    {
+      upsert: true
+    }
+  );
+
+  const expired_ts = now.getTime() - lock_timeout;
+
+  const res = await collection.updateOne(
+    {
+      $and: [
+        { name: params.name },
+        {
+          $or: [{ active_lock: { $exists: false } }, { 'active_lock.ts': { $lte: new Date(expired_ts) } }]
+        }
+      ]
+    },
+    {
+      $set: {
+        active_lock: {
+          lock_id: lock_id,
+          ts: now
+        }
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    return null;
+  }
+
+  return lock_id.toString();
+};
+
+const refreshLock = async (collection: Collection, lock_id: string) => {
+  const lockId = new bson.ObjectId(lock_id);
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lockId
+    },
+    {
+      $set: {
+        'active_lock.ts': new Date()
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not refresh');
+  }
+};
+
+export const releaseLock = async (collection: Collection, lock_id: string) => {
+  const lockId = new bson.ObjectId(lock_id);
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lockId
+    },
+    {
+      $unset: {
+        active_lock: true
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not release');
+  }
+};
+
+export type CreateLockManagerParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL for each lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+export const createMongoLockManager = (
+  collection: Collection,
+  params: CreateLockManagerParams
+): framework.locks.LockManager => {
+  return {
+    acquire: () => acquireLock(collection, params),
+    refresh: (lock_id: string) => refreshLock(collection, lock_id),
+    release: (lock_id: string) => releaseLock(collection, lock_id),
+
+    lock: async (handler) => {
+      const lock_id = await acquireLock(collection, params);
+      if (!lock_id) {
+        throw new framework.locks.LockActiveError();
+      }
+
+      try {
+        await handler(() => refreshLock(collection, lock_id));
+      } finally {
+        await releaseLock(collection, lock_id);
+      }
+    }
+  };
+};

package/src/migrations/MonogMigrationAgent.ts

@@ -0,0 +1,39 @@
+import * as framework from '@powersync/lib-services-framework';
+
+import { migrations } from '@powersync/service-core';
+import { configFile } from '@powersync/service-types';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { createMongoLockManager } from '../locks/MonogLocks.js';
+import { createPowerSyncMongo, PowerSyncMongo } from '../storage/storage-index.js';
+import { createMongoMigrationStore } from './mongo-migration-store.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const MONGO_LOCK_PROCESS = 'migrations';
+const MIGRATIONS_DIR = path.join(__dirname, '/db/migrations');
+
+export class MongoMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent {
+  store: framework.MigrationStore;
+  locks: framework.LockManager;
+
+  protected client: PowerSyncMongo;
+
+  constructor(mongoConfig: configFile.MongoStorageConfig) {
+    super();
+
+    this.client = createPowerSyncMongo(mongoConfig);
+
+    this.store = createMongoMigrationStore(this.client.db);
+    this.locks = createMongoLockManager(this.client.locks, { name: MONGO_LOCK_PROCESS });
+  }
+
+  getInternalScriptsDir(): string {
+    return MIGRATIONS_DIR;
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.client.client.close();
+  }
+}

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -0,0 +1,39 @@
+import { migrations } from '@powersync/service-core';
+import { configFile } from '@powersync/service-types';
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const database = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+  await mongo.waitForAuth(database.db);
+  try {
+    await database.bucket_parameters.createIndex(
+      {
+        'key.g': 1,
+        lookup: 1,
+        _id: 1
+      },
+      { name: 'lookup1' }
+    );
+  } finally {
+    await database.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const database = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+  try {
+    if (await database.bucket_parameters.indexExists('lookup')) {
+      await database.bucket_parameters.dropIndex('lookup1');
+    }
+  } finally {
+    await database.client.close();
+  }
+};

package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts

@@ -0,0 +1,5 @@
+export const up = async () => {
+  // No-op - we don't auto-create sync rules anymore
+};
+
+export const down = async () => {};

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -0,0 +1,105 @@
+import { storage as core_storage, migrations } from '@powersync/service-core';
+import { configFile } from '@powersync/service-types';
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+
+interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
+  /**
+   * True if this is the active sync rules.
+   * requires `snapshot_done == true` and `replicating == true`.
+   */
+  active?: boolean;
+
+  /**
+   * True if this sync rules should be used for replication.
+   *
+   * During reprocessing, there is one sync rules with `replicating = true, active = true`,
+   * and one with `replicating = true, active = false, auto_activate = true`.
+   */
+  replicating?: boolean;
+
+  /**
+   * True if the sync rules should set `active = true` when `snapshot_done` = true.
+   */
+  auto_activate?: boolean;
+}
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+
+  await mongo.waitForAuth(db.db);
+  try {
+    // We keep the old flags for existing deployments still shutting down.
+
+    // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: true,
+        auto_activate: true
+      },
+      { $set: { state: core_storage.SyncRuleState.PROCESSING } }
+    );
+
+    // 2. Snapshot done: `active = true, snapshot_done = true, replicating = true, auto_activate = false`
+    await db.sync_rules.updateMany(
+      {
+        active: true
+      },
+      { $set: { state: core_storage.SyncRuleState.ACTIVE } }
+    );
+
+    // 3. Stopped: `active = false, snapshot_done = true, replicating = false, auto_activate = false`.
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: { $ne: true },
+        auto_activate: { $ne: true }
+      },
+      { $set: { state: core_storage.SyncRuleState.STOP } }
+    );
+
+    const remaining = await db.sync_rules.find({ state: null as any }).toArray();
+    if (remaining.length > 0) {
+      const slots = remaining.map((doc) => doc.slot_name).join(', ');
+      throw new Error(`Invalid state for sync rules: ${slots}`);
+    }
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+  try {
+    await db.sync_rules.updateMany(
+      {
+        state: core_storage.SyncRuleState.ACTIVE
+      },
+      { $set: { active: true, replicating: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        state: core_storage.SyncRuleState.PROCESSING
+      },
+      { $set: { active: false, replicating: true, auto_activate: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        $or: [{ state: core_storage.SyncRuleState.STOP }, { state: core_storage.SyncRuleState.TERMINATED }]
+      },
+      { $set: { active: false, replicating: false, auto_activate: false } }
+    );
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -0,0 +1,38 @@
+import { migrations } from '@powersync/service-core';
+import { configFile } from '@powersync/service-types';
+
+import * as storage from '../../../storage/storage-index.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+
+  try {
+    await db.write_checkpoints.createIndex(
+      {
+        user_id: 1
+      },
+      { name: 'user_id' }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+
+  try {
+    if (await db.write_checkpoints.indexExists('user_id')) {
+      await db.write_checkpoints.dropIndex('user_id');
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts

@@ -0,0 +1,40 @@
+import { migrations } from '@powersync/service-core';
+import { configFile } from '@powersync/service-types';
+import * as storage from '../../../storage/storage-index.js';
+
+const INDEX_NAME = 'user_sync_rule_unique';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+
+  try {
+    await db.custom_write_checkpoints.createIndex(
+      {
+        user_id: 1,
+        sync_rules_id: 1
+      },
+      { name: INDEX_NAME, unique: true }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as configFile.MongoStorageConfig);
+
+  try {
+    if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
+      await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/mongo-migration-store.ts

@@ -0,0 +1,62 @@
+import { migrations } from '@powersync/lib-services-framework';
+import { Db } from 'mongodb';
+import * as path from 'path';
+
+/**
+ * A custom store for node-migrate which is used to save and load migrations that have
+ * been operated on to mongo.
+ */
+export const createMongoMigrationStore = (db: Db): migrations.MigrationStore => {
+  const collection = db.collection<migrations.MigrationState>('migrations');
+
+  return {
+    load: async () => {
+      const state_entry = await collection.findOne();
+      if (!state_entry) {
+        return;
+      }
+
+      const { _id, ...state } = state_entry;
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to save
+       * state as `lastRun`.
+       */
+      let last_run = state.last_run;
+      if ('lastRun' in state) {
+        last_run = (state as any).lastRun;
+      }
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to include the
+       * file extension in migration names. This strips that extension off if it exists
+       */
+      const extension = path.extname(last_run);
+      if (extension) {
+        last_run = last_run.replace(extension, '');
+      }
+
+      return {
+        last_run,
+        log: state.log || []
+      };
+    },
+
+    clear: async () => {
+      await collection.deleteMany({});
+    },
+
+    save: async (state: migrations.MigrationState) => {
+      await collection.replaceOne(
+        {},
+        {
+          last_run: state.last_run,
+          log: state.log
+        },
+        {
+          upsert: true
+        }
+      );
+    }
+  };
+};

package/src/module/MongoModule.ts

@@ -1,11 +1,13 @@
 import { api, ConfigurationFileSyncRulesProvider, replication, system, TearDownOptions } from '@powersync/service-core';
 import { MongoRouteAPIAdapter } from '../api/MongoRouteAPIAdapter.js';
+import { MongoMigrationAgent } from '../migrations/MonogMigrationAgent.js';
+import { ChangeStreamReplicator } from '../replication/ChangeStreamReplicator.js';
 import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactory.js';
 import { MongoErrorRateLimiter } from '../replication/MongoErrorRateLimiter.js';
-import { ChangeStreamReplicator } from '../replication/ChangeStreamReplicator.js';
-import * as types from '../types/types.js';
 import { MongoManager } from '../replication/MongoManager.js';
 import { checkSourceConfiguration } from '../replication/replication-utils.js';
+import { MongoStorageProvider } from '../storage/storage-index.js';
+import * as types from '../types/types.js';
 
 export class MongoModule extends replication.ReplicationModule<types.MongoConnectionConfig> {
   constructor() {
@@ -18,6 +20,13 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
 
   async initialize(context: system.ServiceContextContainer): Promise<void> {
     await super.initialize(context);
+    context.storageEngine.registerProvider(new MongoStorageProvider());
+
+    if (types.isMongoStorageConfig(context.configuration.storage)) {
+      context.migrations.registerMigrationAgent(
+        new MongoMigrationAgent(this.resolveConfig(context.configuration.storage))
+      );
+    }
   }
 
   protected createRouteAPIAdapter(): api.RouteAPI {
@@ -55,9 +64,14 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
   async testConnection(config: types.MongoConnectionConfig): Promise<void> {
     this.decodeConfig(config);
     const normalisedConfig = this.resolveConfig(this.decodedConfig!);
-    const connectionManager = new MongoManager(normalisedConfig);
+    const connectionManager = new MongoManager(normalisedConfig, {
+      // Use short timeouts for testing connections.
+      // Must be < 30s, to ensure we get a proper timeout error.
+      socketTimeoutMS: 5_000,
+      serverSelectionTimeoutMS: 5_000
+    });
     try {
-      return checkSourceConfiguration(connectionManager);
+      return await checkSourceConfiguration(connectionManager);
     } finally {
       await connectionManager.end();
     }

package/src/replication/MongoManager.ts

@@ -8,7 +8,10 @@ export class MongoManager {
   public readonly client: mongo.MongoClient;
   public readonly db: mongo.Db;
 
-  constructor(public options: NormalizedMongoConnectionConfig) {
+  constructor(
+    public options: NormalizedMongoConnectionConfig,
+    overrides?: mongo.MongoClientOptions
+  ) {
     // The pool is lazy - no connections are opened until a query is performed.
     this.client = new mongo.MongoClient(options.uri, {
       auth: {
@@ -28,7 +31,8 @@ export class MongoManager {
       maxPoolSize: 8,
 
       maxConnecting: 3,
-      maxIdleTimeMS: 60_000
+      maxIdleTimeMS: 60_000,
+      ...overrides
     });
     this.db = this.client.db(options.database, {});
   }