@powersync/service-module-mongodb-storage 0.0.0-dev-20250108073049

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -0,0 +1,105 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { storage as core_storage, migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
+  /**
+   * True if this is the active sync rules.
+   * requires `snapshot_done == true` and `replicating == true`.
+   */
+  active?: boolean;
+
+  /**
+   * True if this sync rules should be used for replication.
+   *
+   * During reprocessing, there is one sync rules with `replicating = true, active = true`,
+   * and one with `replicating = true, active = false, auto_activate = true`.
+   */
+  replicating?: boolean;
+
+  /**
+   * True if the sync rules should set `active = true` when `snapshot_done` = true.
+   */
+  auto_activate?: boolean;
+}
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  await lib_mongo.waitForAuth(db.db);
+  try {
+    // We keep the old flags for existing deployments still shutting down.
+
+    // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: true,
+        auto_activate: true
+      },
+      { $set: { state: core_storage.SyncRuleState.PROCESSING } }
+    );
+
+    // 2. Snapshot done: `active = true, snapshot_done = true, replicating = true, auto_activate = false`
+    await db.sync_rules.updateMany(
+      {
+        active: true
+      },
+      { $set: { state: core_storage.SyncRuleState.ACTIVE } }
+    );
+
+    // 3. Stopped: `active = false, snapshot_done = true, replicating = false, auto_activate = false`.
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: { $ne: true },
+        auto_activate: { $ne: true }
+      },
+      { $set: { state: core_storage.SyncRuleState.STOP } }
+    );
+
+    const remaining = await db.sync_rules.find({ state: null as any }).toArray();
+    if (remaining.length > 0) {
+      const slots = remaining.map((doc) => doc.slot_name).join(', ');
+      throw new Error(`Invalid state for sync rules: ${slots}`);
+    }
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+  try {
+    await db.sync_rules.updateMany(
+      {
+        state: core_storage.SyncRuleState.ACTIVE
+      },
+      { $set: { active: true, replicating: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        state: core_storage.SyncRuleState.PROCESSING
+      },
+      { $set: { active: false, replicating: true, auto_activate: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        $or: [{ state: core_storage.SyncRuleState.STOP }, { state: core_storage.SyncRuleState.TERMINATED }]
+      },
+      { $set: { active: false, replicating: false, auto_activate: false } }
+    );
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -0,0 +1,38 @@
+import { migrations } from '@powersync/service-core';
+
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.write_checkpoints.createIndex(
+      {
+        user_id: 1
+      },
+      { name: 'user_id' }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    if (await db.write_checkpoints.indexExists('user_id')) {
+      await db.write_checkpoints.dropIndex('user_id');
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts

@@ -0,0 +1,40 @@
+import { migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+const INDEX_NAME = 'user_sync_rule_unique';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.custom_write_checkpoints.createIndex(
+      {
+        user_id: 1,
+        sync_rules_id: 1
+      },
+      { name: INDEX_NAME, unique: true }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
+      await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/mongo-migration-store.ts

@@ -0,0 +1,62 @@
+import { migrations } from '@powersync/lib-services-framework';
+import { Db } from 'mongodb';
+import * as path from 'path';
+
+/**
+ * A custom store for node-migrate which is used to save and load migrations that have
+ * been operated on to mongo.
+ */
+export const createMongoMigrationStore = (db: Db): migrations.MigrationStore => {
+  const collection = db.collection<migrations.MigrationState>('migrations');
+
+  return {
+    load: async () => {
+      const state_entry = await collection.findOne();
+      if (!state_entry) {
+        return;
+      }
+
+      const { _id, ...state } = state_entry;
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to save
+       * state as `lastRun`.
+       */
+      let last_run = state.last_run;
+      if ('lastRun' in state) {
+        last_run = (state as any).lastRun;
+      }
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to include the
+       * file extension in migration names. This strips that extension off if it exists
+       */
+      const extension = path.extname(last_run);
+      if (extension) {
+        last_run = last_run.replace(extension, '');
+      }
+
+      return {
+        last_run,
+        log: state.log || []
+      };
+    },
+
+    clear: async () => {
+      await collection.deleteMany({});
+    },
+
+    save: async (state: migrations.MigrationState) => {
+      await collection.replaceOne(
+        {},
+        {
+          last_run: state.last_run,
+          log: state.log
+        },
+        {
+          upsert: true
+        }
+      );
+    }
+  };
+};

package/src/module/MongoStorageModule.ts

@@ -0,0 +1,37 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import * as core from '@powersync/service-core';
+import { MongoMigrationAgent } from '../migrations/MongoMigrationAgent.js';
+import { MongoStorageProvider } from '../storage/storage-index.js';
+import * as types from '../types/types.js';
+
+export class MongoStorageModule extends core.modules.AbstractModule {
+  constructor() {
+    super({
+      name: 'MongoDB Storage'
+    });
+  }
+
+  async initialize(context: core.system.ServiceContextContainer): Promise<void> {
+    context.storageEngine.registerProvider(new MongoStorageProvider());
+
+    if (types.isMongoStorageConfig(context.configuration.storage)) {
+      context.migrations.registerMigrationAgent(
+        new MongoMigrationAgent(this.resolveConfig(context.configuration.storage))
+      );
+    }
+  }
+
+  /**
+   * Combines base config with normalized connection settings
+   */
+  private resolveConfig(config: types.MongoStorageConfig) {
+    return {
+      ...config,
+      ...lib_mongo.normalizeMongoConfig(config)
+    };
+  }
+
+  async teardown(options: core.modules.TearDownOptions): Promise<void> {
+    // teardown is implemented in the storage engine
+  }
+}