@powersync/service-core 0.12.2 → 0.14.0

package/src/locks/MongoLocks.ts

@@ -1,142 +0,0 @@
-import * as mongo from 'mongodb';
-import * as bson from 'bson';
-import { LockActiveError, LockManager } from './LockManager.js';
-
-/**
- * Lock Document Schema
- */
-export type Lock = {
-  name: string;
-  active_lock?: {
-    lock_id: bson.ObjectId;
-    ts: Date;
-  };
-};
-
-export type Collection = mongo.Collection<Lock>;
-
-export type AcquireLockParams = {
-  /**
-   * Name of the process/user trying to acquire the lock.
-   */
-  name: string;
-  /**
-   * The TTL of the lock (ms). Default: 60000 ms (1 min)
-   */
-  timeout?: number;
-};
-
-const DEFAULT_LOCK_TIMEOUT = 60 * 1000; // 1 minute
-
-const acquireLock = async (collection: Collection, params: AcquireLockParams) => {
-  const now = new Date();
-  const lock_timeout = params.timeout ?? DEFAULT_LOCK_TIMEOUT;
-  const lock_id = new bson.ObjectId();
-
-  await collection.updateOne(
-    {
-      name: params.name
-    },
-    {
-      $setOnInsert: {
-        name: params.name
-      }
-    },
-    {
-      upsert: true
-    }
-  );
-
-  const expired_ts = now.getTime() - lock_timeout;
-
-  const res = await collection.updateOne(
-    {
-      $and: [
-        { name: params.name },
-        {
-          $or: [{ active_lock: { $exists: false } }, { 'active_lock.ts': { $lte: new Date(expired_ts) } }]
-        }
-      ]
-    },
-    {
-      $set: {
-        active_lock: {
-          lock_id: lock_id,
-          ts: now
-        }
-      }
-    }
-  );
-
-  if (res.modifiedCount === 0) {
-    return null;
-  }
-
-  return lock_id;
-};
-
-const refreshLock = async (collection: Collection, lock_id: bson.ObjectId) => {
-  const res = await collection.updateOne(
-    {
-      'active_lock.lock_id': lock_id
-    },
-    {
-      $set: {
-        'active_lock.ts': new Date()
-      }
-    }
-  );
-
-  if (res.modifiedCount === 0) {
-    throw new Error('Lock not found, could not refresh');
-  }
-};
-
-export const releaseLock = async (collection: Collection, lock_id: bson.ObjectId) => {
-  const res = await collection.updateOne(
-    {
-      'active_lock.lock_id': lock_id
-    },
-    {
-      $unset: {
-        active_lock: true
-      }
-    }
-  );
-
-  if (res.modifiedCount === 0) {
-    throw new Error('Lock not found, could not release');
-  }
-};
-
-export type CreateLockManagerParams = {
-  /**
-   * Name of the process/user trying to acquire the lock.
-   */
-  name: string;
-  /**
-   * The TTL for each lock (ms). Default: 60000 ms (1 min)
-   */
-  timeout?: number;
-};
-
-export const createMongoLockManager = (collection: Collection, params: CreateLockManagerParams): LockManager => {
-  return {
-    acquire: () => acquireLock(collection, params),
-    refresh: (lock_id: bson.ObjectId) => refreshLock(collection, lock_id),
-    release: (lock_id: bson.ObjectId) => releaseLock(collection, lock_id),
-
-    lock: async (handler) => {
-      const lock_id = await acquireLock(collection, params);
-      if (!lock_id) {
-        throw new LockActiveError();
-      }
-
-      try {
-        await handler(() => refreshLock(collection, lock_id));
-      } finally {
-        await releaseLock(collection, lock_id);
-      }
-    }
-  };
-};

package/src/locks/locks-index.ts

@@ -1,2 +0,0 @@
-export * from './LockManager.js';
-export * from './MongoLocks.js';

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -1,38 +0,0 @@
-import * as mongo from '../../../db/mongo.js';
-import * as storage from '../../../storage/storage-index.js';
-import * as utils from '../../../util/util-index.js';
-
-export const up = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-
-  const config = await utils.loadConfig(runner_config);
-
-  const database = storage.createPowerSyncMongo(config.storage);
-  await mongo.waitForAuth(database.db);
-  try {
-    await database.bucket_parameters.createIndex(
-      {
-        'key.g': 1,
-        lookup: 1,
-        _id: 1
-      },
-      { name: 'lookup1' }
-    );
-  } finally {
-    await database.client.close();
-  }
-};
-
-export const down = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-  const config = await utils.loadConfig(runner_config);
-
-  const database = storage.createPowerSyncMongo(config.storage);
-  try {
-    if (await database.bucket_parameters.indexExists('lookup')) {
-      await database.bucket_parameters.dropIndex('lookup1');
-    }
-  } finally {
-    await database.client.close();
-  }
-};

package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts

@@ -1,5 +0,0 @@
-export const up = async () => {
-  // No-op - we don't auto-create sync rules anymore
-};
-
-export const down = async () => {};

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -1,102 +0,0 @@
-import * as mongo from '../../../db/mongo.js';
-import * as storage from '../../../storage/storage-index.js';
-import * as utils from '../../../util/util-index.js';
-
-interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
-  /**
-   * True if this is the active sync rules.
-   * requires `snapshot_done == true` and `replicating == true`.
-   */
-  active?: boolean;
-
-  /**
-   * True if this sync rules should be used for replication.
-   *
-   * During reprocessing, there is one sync rules with `replicating = true, active = true`,
-   * and one with `replicating = true, active = false, auto_activate = true`.
-   */
-  replicating?: boolean;
-
-  /**
-   * True if the sync rules should set `active = true` when `snapshot_done` = true.
-   */
-  auto_activate?: boolean;
-}
-
-export const up = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-  const config = await utils.loadConfig(runner_config);
-  const db = storage.createPowerSyncMongo(config.storage);
-
-  await mongo.waitForAuth(db.db);
-  try {
-    // We keep the old flags for existing deployments still shutting down.
-
-    // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
-    await db.sync_rules.updateMany(
-      {
-        active: { $ne: true },
-        replicating: true,
-        auto_activate: true
-      },
-      { $set: { state: storage.SyncRuleState.PROCESSING } }
-    );
-
-    // 2. Snapshot done: `active = true, snapshot_done = true, replicating = true, auto_activate = false`
-    await db.sync_rules.updateMany(
-      {
-        active: true
-      },
-      { $set: { state: storage.SyncRuleState.ACTIVE } }
-    );
-
-    // 3. Stopped: `active = false, snapshot_done = true, replicating = false, auto_activate = false`.
-    await db.sync_rules.updateMany(
-      {
-        active: { $ne: true },
-        replicating: { $ne: true },
-        auto_activate: { $ne: true }
-      },
-      { $set: { state: storage.SyncRuleState.STOP } }
-    );
-
-    const remaining = await db.sync_rules.find({ state: null as any }).toArray();
-    if (remaining.length > 0) {
-      const slots = remaining.map((doc) => doc.slot_name).join(', ');
-      throw new Error(`Invalid state for sync rules: ${slots}`);
-    }
-  } finally {
-    await db.client.close();
-  }
-};
-
-export const down = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-  const config = await utils.loadConfig(runner_config);
-
-  const db = storage.createPowerSyncMongo(config.storage);
-  try {
-    await db.sync_rules.updateMany(
-      {
-        state: storage.SyncRuleState.ACTIVE
-      },
-      { $set: { active: true, replicating: true } }
-    );
-
-    await db.sync_rules.updateMany(
-      {
-        state: storage.SyncRuleState.PROCESSING
-      },
-      { $set: { active: false, replicating: true, auto_activate: true } }
-    );
-
-    await db.sync_rules.updateMany(
-      {
-        $or: [{ state: storage.SyncRuleState.STOP }, { state: storage.SyncRuleState.TERMINATED }]
-      },
-      { $set: { active: false, replicating: false, auto_activate: false } }
-    );
-  } finally {
-    await db.client.close();
-  }
-};

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -1,34 +0,0 @@
-import * as storage from '../../../storage/storage-index.js';
-import * as utils from '../../../util/util-index.js';
-
-export const up = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-  const config = await utils.loadConfig(runner_config);
-  const db = storage.createPowerSyncMongo(config.storage);
-
-  try {
-    await db.write_checkpoints.createIndex(
-      {
-        user_id: 1
-      },
-      { name: 'user_id' }
-    );
-  } finally {
-    await db.client.close();
-  }
-};
-
-export const down = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-  const config = await utils.loadConfig(runner_config);
-
-  const db = storage.createPowerSyncMongo(config.storage);
-
-  try {
-    if (await db.write_checkpoints.indexExists('user_id')) {
-      await db.write_checkpoints.dropIndex('user_id');
-    }
-  } finally {
-    await db.client.close();
-  }
-};

package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts

@@ -1,37 +0,0 @@
-import * as storage from '../../../storage/storage-index.js';
-import * as utils from '../../../util/util-index.js';
-
-const INDEX_NAME = 'user_sync_rule_unique';
-
-export const up = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-  const config = await utils.loadConfig(runner_config);
-  const db = storage.createPowerSyncMongo(config.storage);
-
-  try {
-    await db.custom_write_checkpoints.createIndex(
-      {
-        user_id: 1,
-        sync_rules_id: 1
-      },
-      { name: INDEX_NAME, unique: true }
-    );
-  } finally {
-    await db.client.close();
-  }
-};
-
-export const down = async (context: utils.MigrationContext) => {
-  const { runner_config } = context;
-  const config = await utils.loadConfig(runner_config);
-
-  const db = storage.createPowerSyncMongo(config.storage);
-
-  try {
-    if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
-      await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
-    }
-  } finally {
-    await db.client.close();
-  }
-};

package/src/migrations/definitions.ts

@@ -1,21 +0,0 @@
-export type Migration = {
-  name: string;
-  up: () => Promise<void>;
-  down: () => Promise<void>;
-};
-
-export enum Direction {
-  Up = 'up',
-  Down = 'down'
-}
-
-export type ExecutedMigration = {
-  name: string;
-  direction: Direction;
-  timestamp: Date;
-};
-
-export type MigrationState = {
-  last_run: string;
-  log: ExecutedMigration[];
-};

package/src/migrations/executor.ts

@@ -1,87 +0,0 @@
-import { logger } from '@powersync/lib-services-framework';
-import * as defs from './definitions.js';
-import { MigrationStore } from './store/migration-store.js';
-
-type ExecuteParams = {
-  migrations: defs.Migration[];
-  state?: defs.MigrationState;
-
-  direction: defs.Direction;
-  count?: number;
-};
-
-export async function* execute(params: ExecuteParams): AsyncGenerator<defs.ExecutedMigration> {
-  let migrations = [...params.migrations];
-  if (params.direction === defs.Direction.Down) {
-    migrations = migrations.reverse();
-  }
-
-  let index = 0;
-
-  if (params.state) {
-    // Find the index of the last run
-    index = migrations.findIndex((migration) => {
-      return migration.name === params.state!.last_run;
-    });
-
-    if (index === -1) {
-      throw new Error(`The last run migration ${params.state?.last_run} was not found in the given set of migrations`);
-    }
-
-    // If we are migrating down then we want to include the last run migration, otherwise we want to start at the next one
-    if (params.direction === defs.Direction.Up) {
-      index += 1;
-    }
-  }
-
-  migrations = migrations.slice(index);
-
-  let i = 0;
-  for (const migration of migrations) {
-    if (params.count && params.count === i) {
-      return;
-    }
-
-    logger.info(`Executing ${migration.name} (${params.direction})`);
-    try {
-      switch (params.direction) {
-        case defs.Direction.Up: {
-          await migration.up();
-          break;
-        }
-        case defs.Direction.Down: {
-          await migration.down();
-          break;
-        }
-      }
-      logger.debug(`Success`);
-    } catch (err) {
-      logger.error(`Failed`, err);
-      process.exit(1);
-    }
-
-    yield {
-      name: migration.name,
-      direction: params.direction,
-      timestamp: new Date()
-    };
-
-    i++;
-  }
-}
-
-type WriteLogsParams = {
-  store: MigrationStore;
-  state?: defs.MigrationState;
-  log_stream: Iterable<defs.ExecutedMigration> | AsyncIterable<defs.ExecutedMigration>;
-};
-export const writeLogsToStore = async (params: WriteLogsParams): Promise<void> => {
-  const log = [...(params.state?.log || [])];
-  for await (const migration of params.log_stream) {
-    log.push(migration);
-    await params.store.save({
-      last_run: migration.name,
-      log: log
-    });
-  }
-};

package/src/migrations/migrations.ts

@@ -1,142 +0,0 @@
-import * as fs from 'fs/promises';
-import * as path from 'path';
-import { fileURLToPath } from 'url';
-
-import { logger } from '@powersync/lib-services-framework';
-import * as db from '../db/db-index.js';
-import * as locks from '../locks/locks-index.js';
-import * as util from '../util/util-index.js';
-import { Direction } from './definitions.js';
-import { execute, writeLogsToStore } from './executor.js';
-import { createMongoMigrationStore } from './store/migration-store.js';
-
-const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
-const MONGO_LOCK_PROCESS = 'migrations';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const MIGRATIONS_DIR = path.join(__dirname, '/db/migrations');
-
-export type MigrationOptions = {
-  direction: Direction;
-  runner_config: util.RunnerConfig;
-};
-
-export type AutomaticMigrationParams = {
-  config: util.ResolvedPowerSyncConfig;
-  runner_config: util.RunnerConfig;
-};
-
-/**
- * Loads migrations and injects a custom context for loading the specified
- * runner configuration.
- */
-const loadMigrations = async (dir: string, runnerConfig: util.RunnerConfig) => {
-  const files = await fs.readdir(dir);
-  const migrations = files.filter((file) => {
-    return path.extname(file) === '.js';
-  });
-
-  const context: util.MigrationContext = {
-    runner_config: runnerConfig
-  };
-
-  return await Promise.all(
-    migrations.map(async (migration) => {
-      const module = await import(path.resolve(dir, migration));
-      return {
-        name: path.basename(migration).replace(path.extname(migration), ''),
-        up: () => module.up(context),
-        down: () => module.down(context)
-      };
-    })
-  );
-};
-
-/**
- * Runs migration scripts exclusively using Mongo locks
- */
-export const migrate = async (options: MigrationOptions) => {
-  const { direction, runner_config } = options;
-
-  const config = await util.loadConfig(runner_config);
-  const { storage } = config;
-  /**
-   * Try and get Mongo from config file.
-   * But this might not be available in Journey Micro as we use the standard Mongo.
-   */
-
-  const client = db.mongo.createMongoClient(storage);
-  logger.info('Connecting to MongoDB');
-  await client.connect();
-
-  const clientDB = client.db(storage.database);
-  const collection = clientDB.collection<locks.Lock>(DEFAULT_MONGO_LOCK_COLLECTION);
-
-  const manager = locks.createMongoLockManager(collection, {
-    name: MONGO_LOCK_PROCESS
-  });
-
-  // Only one process should execute this at a time.
-  logger.info('Acquiring lock');
-  const lockId = await manager.acquire();
-
-  if (!lockId) {
-    throw new Error('Could not acquire lock_id');
-  }
-
-  let isReleased = false;
-  const releaseLock = async () => {
-    if (isReleased) {
-      return;
-    }
-    await manager.release(lockId);
-    isReleased = true;
-  };
-
-  // For the case where the migration is terminated
-  process.addListener('beforeExit', releaseLock);
-
-  try {
-    logger.info('Loading migrations');
-    const migrations = await loadMigrations(MIGRATIONS_DIR, runner_config);
-
-    // Use the provided config to connect to Mongo
-    const store = createMongoMigrationStore(clientDB);
-
-    const state = await store.load();
-
-    logger.info('Running migrations');
-    const logStream = execute({
-      direction: direction,
-      migrations,
-      state
-    });
-
-    await writeLogsToStore({
-      log_stream: logStream,
-      store,
-      state
-    });
-  } finally {
-    logger.info('Releasing lock');
-    await releaseLock();
-    logger.info('Closing database');
-    await client.close(true);
-    process.removeListener('beforeExit', releaseLock);
-    logger.info('Done with migrations');
-  }
-};
-
-/**
- * Ensures automatic migrations are executed
- */
-export const ensureAutomaticMigrations = async (params: AutomaticMigrationParams) => {
-  if (!params.config.migrations?.disable_auto_migration) {
-    await migrate({
-      direction: Direction.Up,
-      runner_config: params.runner_config
-    });
-  }
-};

package/src/migrations/store/migration-store.ts

@@ -1,63 +0,0 @@
-import { Db } from 'mongodb';
-import * as path from 'path';
-import * as defs from '../definitions.js';
-
-export type MigrationStore = {
-  load: () => Promise<defs.MigrationState | undefined>;
-  save: (state: defs.MigrationState) => Promise<void>;
-};
-
-/**
- * A custom store for node-migrate which is used to save and load migrations that have
- * been operated on to mongo.
- */
-export const createMongoMigrationStore = (db: Db): MigrationStore => {
-  const collection = db.collection<defs.MigrationState>('migrations');
-
-  return {
-    load: async () => {
-      const state_entry = await collection.findOne();
-      if (!state_entry) {
-        return;
-      }
-
-      const { _id, ...state } = state_entry;
-
-      /**
-       * This is for backwards compatibility. A previous version of the migration tool used to save
-       * state as `lastRun`.
-       */
-      let last_run = state.last_run;
-      if ('lastRun' in state) {
-        last_run = (state as any).lastRun;
-      }
-
-      /**
-       * This is for backwards compatibility. A previous version of the migration tool used to include the
-       * file extension in migration names. This strips that extension off if it exists
-       */
-      const extension = path.extname(last_run);
-      if (extension) {
-        last_run = last_run.replace(extension, '');
-      }
-
-      return {
-        last_run,
-        log: state.log || []
-      };
-    },
-
-    save: async (state: defs.MigrationState) => {
-      await collection.replaceOne(
-        {},
-        {
-          last_run: state.last_run,
-          log: state.log
-        },
-        {
-          upsert: true
-        }
-      );
-    }
-  };
-};