@powersync/service-core 0.2.2 → 0.4.0

package/src/entry/cli-entry.ts

@@ -1,11 +1,10 @@
 import { Command } from 'commander';
 
-import * as micro from '@journeyapps-platform/micro';
-
-import * as utils from '@/util/util-index.js';
+import * as utils from '../util/util-index.js';
 import { registerMigrationAction } from './commands/migrate-action.js';
 import { registerTearDownAction } from './commands/teardown-action.js';
 import { registerStartAction } from './entry-index.js';
+import { logger } from '@powersync/lib-services-framework';
 
 /**
  * Generates a Commander program which serves as the entry point
@@ -33,7 +32,7 @@ export function generateEntryProgram(startHandlers?: Record<utils.ServiceRunner,
       try {
         await entryProgram.parseAsync();
       } catch (e) {
-        micro.logger.error('Fatal error', e);
+        logger.error('Fatal error', e);
         process.exit(1);
       }
     }

package/src/entry/commands/config-command.ts

@@ -1,6 +1,6 @@
 import { Command } from 'commander';
 
-import * as util from '@/util/util-index.js';
+import * as util from '../../util/util-index.js';
 
 /**
  * Wraps a Command with the standard config options

package/src/entry/commands/migrate-action.ts

@@ -1,8 +1,9 @@
 import { Command } from 'commander';
-import { Direction } from '@journeyapps-platform/micro-migrate';
 
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
-import { migrate } from '@/migrations/migrations.js';
+import { migrate } from '../../migrations/migrations.js';
+import { Direction } from '../../migrations/definitions.js';
+import { logger } from '@powersync/lib-services-framework';
 
 const COMMAND_NAME = 'migrate';
 
@@ -17,9 +18,16 @@ export function registerMigrationAction(program: Command) {
     .action(async (direction: Direction, options) => {
       const runnerConfig = extractRunnerOptions(options);
 
-      await migrate({
-        direction,
-        runner_config: runnerConfig
-      });
+      try {
+        await migrate({
+          direction,
+          runner_config: runnerConfig
+        });
+
+        process.exit(0);
+      } catch (e) {
+        logger.error(`Migration failure`, e);
+        process.exit(1);
+      }
     });
 }

package/src/entry/commands/start-action.ts

@@ -1,6 +1,6 @@
 import { Command } from 'commander';
 
-import * as utils from '@/util/util-index.js';
+import * as utils from '../../util/util-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'start';

package/src/entry/commands/teardown-action.ts

@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
-import { teardown } from '@/runner/teardown.js';
+import { teardown } from '../../runner/teardown.js';
 
 const COMMAND_NAME = 'teardown';
 

package/src/index.ts

@@ -12,11 +12,14 @@ export * as db from './db/db-index.js';
 export * from './entry/entry-index.js';
 export * as entry from './entry/entry-index.js';
 
+// Re-export framework for easy use of Container API
+export * as framework from '@powersync/lib-services-framework';
+
 export * from './metrics/Metrics.js';
 export * as metrics from './metrics/Metrics.js';
 
 export * from './migrations/migrations.js';
-export * as migrations from './migrations/migrations.js';
+export * as migrations from './migrations/migrations-index.js';
 
 export * from './replication/replication-index.js';
 export * as replication from './replication/replication-index.js';
@@ -31,7 +34,7 @@ export * from './sync/sync-index.js';
 export * as sync from './sync/sync-index.js';
 
 export * from './system/CorePowerSyncSystem.js';
-export * as system from './system/CorePowerSyncSystem.js';
+export * as system from './system/system-index.js';
 
 export * from './util/util-index.js';
 export * as utils from './util/util-index.js';

package/src/locks/LockManager.ts

@@ -0,0 +1,16 @@
+import * as bson from 'bson';
+
+export class LockActiveError extends Error {
+  constructor() {
+    super('Lock is already active');
+    this.name = this.constructor.name;
+  }
+}
+
+export type LockManager = {
+  acquire: () => Promise<bson.ObjectId | null>;
+  refresh: (lock_id: bson.ObjectId) => Promise<void>;
+  release: (lock_id: bson.ObjectId) => Promise<void>;
+
+  lock: (handler: (refresh: () => Promise<void>) => Promise<void>) => Promise<void>;
+};

package/src/locks/MongoLocks.ts

@@ -0,0 +1,142 @@
+import * as mongo from 'mongodb';
+import * as bson from 'bson';
+import { LockActiveError, LockManager } from './LockManager.js';
+
+/**
+ * Lock Document Schema
+ */
+export type Lock = {
+  name: string;
+  active_lock?: {
+    lock_id: bson.ObjectId;
+    ts: Date;
+  };
+};
+
+export type Collection = mongo.Collection<Lock>;
+
+export type AcquireLockParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL of the lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+const DEFAULT_LOCK_TIMEOUT = 60 * 1000; // 1 minute
+
+const acquireLock = async (collection: Collection, params: AcquireLockParams) => {
+  const now = new Date();
+  const lock_timeout = params.timeout ?? DEFAULT_LOCK_TIMEOUT;
+  const lock_id = new bson.ObjectId();
+
+  await collection.updateOne(
+    {
+      name: params.name
+    },
+    {
+      $setOnInsert: {
+        name: params.name
+      }
+    },
+    {
+      upsert: true
+    }
+  );
+
+  const expired_ts = now.getTime() - lock_timeout;
+
+  const res = await collection.updateOne(
+    {
+      $and: [
+        { name: params.name },
+        {
+          $or: [{ active_lock: { $exists: false } }, { 'active_lock.ts': { $lte: new Date(expired_ts) } }]
+        }
+      ]
+    },
+    {
+      $set: {
+        active_lock: {
+          lock_id: lock_id,
+          ts: now
+        }
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    return null;
+  }
+
+  return lock_id;
+};
+
+const refreshLock = async (collection: Collection, lock_id: bson.ObjectId) => {
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lock_id
+    },
+    {
+      $set: {
+        'active_lock.ts': new Date()
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not refresh');
+  }
+};
+
+export const releaseLock = async (collection: Collection, lock_id: bson.ObjectId) => {
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lock_id
+    },
+    {
+      $unset: {
+        active_lock: true
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not release');
+  }
+};
+
+export type CreateLockManagerParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL for each lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+export const createMongoLockManager = (collection: Collection, params: CreateLockManagerParams): LockManager => {
+  return {
+    acquire: () => acquireLock(collection, params),
+    refresh: (lock_id: bson.ObjectId) => refreshLock(collection, lock_id),
+    release: (lock_id: bson.ObjectId) => releaseLock(collection, lock_id),
+
+    lock: async (handler) => {
+      const lock_id = await acquireLock(collection, params);
+      if (!lock_id) {
+        throw new LockActiveError();
+      }
+
+      try {
+        await handler(() => refreshLock(collection, lock_id));
+      } finally {
+        await releaseLock(collection, lock_id);
+      }
+    }
+  };
+};

package/src/locks/locks-index.ts

@@ -0,0 +1,2 @@
+export * from './LockManager.js';
+export * from './MongoLocks.js';

package/src/metrics/Metrics.ts

@@ -1,13 +1,13 @@
-import * as micro from '@journeyapps-platform/micro';
 import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
 import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
 import * as jpgwire from '@powersync/service-jpgwire';
-import * as util from '@/util/util-index.js';
-import * as storage from '@/storage/storage-index.js';
+import * as util from '../util/util-index.js';
+import * as storage from '../storage/storage-index.js';
 import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
 import { Resource } from '@opentelemetry/resources';
+import { logger } from '@powersync/lib-services-framework';
 
 export interface MetricsOptions {
   disable_telemetry_sharing: boolean;
@@ -144,9 +144,9 @@ export class Metrics {
     if (Metrics.instance) {
       return;
     }
-    micro.logger.info('Configuring telemetry.');
+    logger.info('Configuring telemetry.');
 
-    micro.logger.info(
+    logger.info(
       `
 Attention:
 PowerSync collects completely anonymous telemetry regarding usage.
@@ -164,7 +164,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
     configuredExporters.push(prometheusExporter);
 
     if (!options.disable_telemetry_sharing) {
-      micro.logger.info('Sharing anonymous telemetry');
+      logger.info('Sharing anonymous telemetry');
       const periodicExporter = new PeriodicExportingMetricReader({
         exporter: new OTLPMetricExporter({
           url: options.internal_metrics_endpoint
@@ -189,7 +189,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
 
     Metrics.instance = new Metrics(meterProvider, prometheusExporter);
 
-    micro.logger.info('Telemetry configuration complete.');
+    logger.info('Telemetry configuration complete.');
   }
 
   public async shutdown(): Promise<void> {
@@ -212,7 +212,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
     function getMetrics() {
       if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
         cachedRequest = system.storage.getStorageMetrics().catch((e) => {
-          micro.logger.error(`Failed to get storage metrics`, e);
+          logger.error(`Failed to get storage metrics`, e);
           return null;
         });
         cacheTimestamp = Date.now();

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -1,6 +1,6 @@
-import * as mongo from '@/db/mongo.js';
-import * as storage from '@/storage/storage-index.js';
-import * as utils from '@/util/util-index.js';
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
 
 export const up = async (context?: utils.MigrationContext) => {
   const config = await utils.loadConfig(context?.runner_config);

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -1,6 +1,6 @@
-import * as mongo from '@/db/mongo.js';
-import * as storage from '@/storage/storage-index.js';
-import * as utils from '@/util/util-index.js';
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
 
 interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   /**

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -1,5 +1,5 @@
-import * as storage from '@/storage/storage-index.js';
-import * as utils from '@/util/util-index.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
 
 export const up = async (context?: utils.MigrationContext) => {
   const config = await utils.loadConfig(context?.runner_config);

package/src/migrations/definitions.ts

@@ -0,0 +1,21 @@
+export type Migration = {
+  name: string;
+  up: () => Promise<void>;
+  down: () => Promise<void>;
+};
+
+export enum Direction {
+  Up = 'up',
+  Down = 'down'
+}
+
+export type ExecutedMigration = {
+  name: string;
+  direction: Direction;
+  timestamp: Date;
+};
+
+export type MigrationState = {
+  last_run: string;
+  log: ExecutedMigration[];
+};

package/src/migrations/executor.ts

@@ -0,0 +1,87 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as defs from './definitions.js';
+import { MigrationStore } from './store/migration-store.js';
+
+type ExecuteParams = {
+  migrations: defs.Migration[];
+  state?: defs.MigrationState;
+
+  direction: defs.Direction;
+  count?: number;
+};
+
+export async function* execute(params: ExecuteParams): AsyncGenerator<defs.ExecutedMigration> {
+  let migrations = [...params.migrations];
+  if (params.direction === defs.Direction.Down) {
+    migrations = migrations.reverse();
+  }
+
+  let index = 0;
+
+  if (params.state) {
+    // Find the index of the last run
+    index = migrations.findIndex((migration) => {
+      return migration.name === params.state!.last_run;
+    });
+
+    if (index === -1) {
+      throw new Error(`The last run migration ${params.state?.last_run} was not found in the given set of migrations`);
+    }
+
+    // If we are migrating down then we want to include the last run migration, otherwise we want to start at the next one
+    if (params.direction === defs.Direction.Up) {
+      index += 1;
+    }
+  }
+
+  migrations = migrations.slice(index);
+
+  let i = 0;
+  for (const migration of migrations) {
+    if (params.count && params.count === i) {
+      return;
+    }
+
+    logger.info(`Executing ${migration.name} (${params.direction})`);
+    try {
+      switch (params.direction) {
+        case defs.Direction.Up: {
+          await migration.up();
+          break;
+        }
+        case defs.Direction.Down: {
+          await migration.down();
+          break;
+        }
+      }
+      logger.debug(`Success`);
+    } catch (err) {
+      logger.error(`Failed`, err);
+      process.exit(1);
+    }
+
+    yield {
+      name: migration.name,
+      direction: params.direction,
+      timestamp: new Date()
+    };
+
+    i++;
+  }
+}
+
+type WriteLogsParams = {
+  store: MigrationStore;
+  state?: defs.MigrationState;
+  log_stream: Iterable<defs.ExecutedMigration> | AsyncIterable<defs.ExecutedMigration>;
+};
+export const writeLogsToStore = async (params: WriteLogsParams): Promise<void> => {
+  const log = [...(params.state?.log || [])];
+  for await (const migration of params.log_stream) {
+    log.push(migration);
+    await params.store.save({
+      last_run: migration.name,
+      log: log
+    });
+  }
+};

package/src/migrations/migrations-index.ts

@@ -0,0 +1,3 @@
+export * from './definitions.js';
+export * from './executor.js';
+export * from './migrations.js';

package/src/migrations/migrations.ts

@@ -1,12 +1,14 @@
-import { locks } from '@journeyapps-platform/micro';
 import * as fs from 'fs/promises';
 import * as path from 'path';
 import { fileURLToPath } from 'url';
 
-import { Direction, createMongoMigrationStore, execute, writeLogsToStore } from '@journeyapps-platform/micro-migrate';
-
-import * as db from '@/db/db-index.js';
-import * as util from '@/util/util-index.js';
+import * as db from '../db/db-index.js';
+import * as util from '../util/util-index.js';
+import * as locks from '../locks/locks-index.js';
+import { Direction } from './definitions.js';
+import { createMongoMigrationStore } from './store/migration-store.js';
+import { execute, writeLogsToStore } from './executor.js';
+import { logger } from '@powersync/lib-services-framework';
 
 const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
 const MONGO_LOCK_PROCESS = 'migrations';
@@ -62,6 +64,7 @@ export const migrate = async (options: MigrationOptions) => {
   const { storage } = config;
 
   const client = db.mongo.createMongoClient(storage);
+  logger.info('Connecting to MongoDB');
   await client.connect();
 
   const clientDB = client.db(storage.database);
@@ -72,6 +75,7 @@ export const migrate = async (options: MigrationOptions) => {
   });
 
   // Only one process should execute this at a time.
+  logger.info('Acquiring lock');
   const lockId = await manager.acquire();
 
   if (!lockId) {
@@ -91,18 +95,15 @@ export const migrate = async (options: MigrationOptions) => {
   process.addListener('beforeExit', releaseLock);
 
   try {
+    logger.info('Loading migrations');
     const migrations = await loadMigrations(MIGRATIONS_DIR, runner_config);
 
     // Use the provided config to connect to Mongo
-    const store = createMongoMigrationStore({
-      uri: storage.uri,
-      database: storage.database,
-      username: storage.username,
-      password: storage.password
-    });
+    const store = createMongoMigrationStore(clientDB);
 
     const state = await store.load();
 
+    logger.info('Running migrations');
     const logStream = execute({
       direction: direction,
       migrations,
@@ -115,8 +116,11 @@ export const migrate = async (options: MigrationOptions) => {
       state
     });
   } finally {
+    logger.info('Releasing lock');
     await releaseLock();
+    logger.info('Closing database');
     await client.close(true);
     process.removeListener('beforeExit', releaseLock);
+    logger.info('Done with migrations');
   }
 };

package/src/migrations/store/migration-store.ts

@@ -0,0 +1,63 @@
+import { Db } from 'mongodb';
+import * as path from 'path';
+import * as defs from '../definitions.js';
+
+export type MigrationStore = {
+  load: () => Promise<defs.MigrationState | undefined>;
+  save: (state: defs.MigrationState) => Promise<void>;
+};
+
+/**
+ * A custom store for node-migrate which is used to save and load migrations that have
+ * been operated on to mongo.
+ */
+export const createMongoMigrationStore = (db: Db): MigrationStore => {
+  const collection = db.collection<defs.MigrationState>('migrations');
+
+  return {
+    load: async () => {
+      const state_entry = await collection.findOne();
+      if (!state_entry) {
+        return;
+      }
+
+      const { _id, ...state } = state_entry;
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to save
+       * state as `lastRun`.
+       */
+      let last_run = state.last_run;
+      if ('lastRun' in state) {
+        last_run = (state as any).lastRun;
+      }
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to include the
+       * file extension in migration names. This strips that extension off if it exists
+       */
+      const extension = path.extname(last_run);
+      if (extension) {
+        last_run = last_run.replace(extension, '');
+      }
+
+      return {
+        last_run,
+        log: state.log || []
+      };
+    },
+
+    save: async (state: defs.MigrationState) => {
+      await collection.replaceOne(
+        {},
+        {
+          last_run: state.last_run,
+          log: state.log
+        },
+        {
+          upsert: true
+        }
+      );
+    }
+  };
+};

package/src/replication/WalConnection.ts

@@ -3,8 +3,8 @@ import { pgwireRows } from '@powersync/service-jpgwire';
 import { DEFAULT_TAG, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
 import { ReplicationError, TableInfo } from '@powersync/service-types';
 
-import * as storage from '@/storage/storage-index.js';
-import * as util from '@/util/util-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 import { ReplicaIdentityResult, getReplicationIdentityColumns } from './util.js';
 /**