@powersync/service-core 0.2.1 → 0.3.0

package/src/index.ts

@@ -12,11 +12,14 @@ export * as db from './db/db-index.js';
 export * from './entry/entry-index.js';
 export * as entry from './entry/entry-index.js';
 
+// Re-export framework for easy use of Container API
+export * as framework from '@powersync/lib-services-framework';
+
 export * from './metrics/Metrics.js';
 export * as metrics from './metrics/Metrics.js';
 
 export * from './migrations/migrations.js';
-export * as migrations from './migrations/migrations.js';
+export * as migrations from './migrations/migrations-index.js';
 
 export * from './replication/replication-index.js';
 export * as replication from './replication/replication-index.js';
@@ -31,7 +34,7 @@ export * from './sync/sync-index.js';
 export * as sync from './sync/sync-index.js';
 
 export * from './system/CorePowerSyncSystem.js';
-export * as system from './system/CorePowerSyncSystem.js';
+export * as system from './system/system-index.js';
 
 export * from './util/util-index.js';
 export * as utils from './util/util-index.js';

package/src/locks/LockManager.ts

@@ -0,0 +1,16 @@
+import * as bson from 'bson';
+
+export class LockActiveError extends Error {
+  constructor() {
+    super('Lock is already active');
+    this.name = this.constructor.name;
+  }
+}
+
+export type LockManager = {
+  acquire: () => Promise<bson.ObjectId | null>;
+  refresh: (lock_id: bson.ObjectId) => Promise<void>;
+  release: (lock_id: bson.ObjectId) => Promise<void>;
+
+  lock: (handler: (refresh: () => Promise<void>) => Promise<void>) => Promise<void>;
+};

package/src/locks/MongoLocks.ts

@@ -0,0 +1,142 @@
+import * as mongo from 'mongodb';
+import * as bson from 'bson';
+import { LockActiveError, LockManager } from './LockManager.js';
+
+/**
+ * Lock Document Schema
+ */
+export type Lock = {
+  name: string;
+  active_lock?: {
+    lock_id: bson.ObjectId;
+    ts: Date;
+  };
+};
+
+export type Collection = mongo.Collection<Lock>;
+
+export type AcquireLockParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL of the lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+const DEFAULT_LOCK_TIMEOUT = 60 * 1000; // 1 minute
+
+const acquireLock = async (collection: Collection, params: AcquireLockParams) => {
+  const now = new Date();
+  const lock_timeout = params.timeout ?? DEFAULT_LOCK_TIMEOUT;
+  const lock_id = new bson.ObjectId();
+
+  await collection.updateOne(
+    {
+      name: params.name
+    },
+    {
+      $setOnInsert: {
+        name: params.name
+      }
+    },
+    {
+      upsert: true
+    }
+  );
+
+  const expired_ts = now.getTime() - lock_timeout;
+
+  const res = await collection.updateOne(
+    {
+      $and: [
+        { name: params.name },
+        {
+          $or: [{ active_lock: { $exists: false } }, { 'active_lock.ts': { $lte: new Date(expired_ts) } }]
+        }
+      ]
+    },
+    {
+      $set: {
+        active_lock: {
+          lock_id: lock_id,
+          ts: now
+        }
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    return null;
+  }
+
+  return lock_id;
+};
+
+const refreshLock = async (collection: Collection, lock_id: bson.ObjectId) => {
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lock_id
+    },
+    {
+      $set: {
+        'active_lock.ts': new Date()
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not refresh');
+  }
+};
+
+export const releaseLock = async (collection: Collection, lock_id: bson.ObjectId) => {
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lock_id
+    },
+    {
+      $unset: {
+        active_lock: true
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not release');
+  }
+};
+
+export type CreateLockManagerParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL for each lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+export const createMongoLockManager = (collection: Collection, params: CreateLockManagerParams): LockManager => {
+  return {
+    acquire: () => acquireLock(collection, params),
+    refresh: (lock_id: bson.ObjectId) => refreshLock(collection, lock_id),
+    release: (lock_id: bson.ObjectId) => releaseLock(collection, lock_id),
+
+    lock: async (handler) => {
+      const lock_id = await acquireLock(collection, params);
+      if (!lock_id) {
+        throw new LockActiveError();
+      }
+
+      try {
+        await handler(() => refreshLock(collection, lock_id));
+      } finally {
+        await releaseLock(collection, lock_id);
+      }
+    }
+  };
+};

package/src/locks/locks-index.ts

@@ -0,0 +1,2 @@
+export * from './LockManager.js';
+export * from './MongoLocks.js';

package/src/metrics/Metrics.ts

@@ -1,13 +1,13 @@
-import * as micro from '@journeyapps-platform/micro';
 import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
 import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
 import * as jpgwire from '@powersync/service-jpgwire';
-import * as util from '@/util/util-index.js';
-import * as storage from '@/storage/storage-index.js';
+import * as util from '../util/util-index.js';
+import * as storage from '../storage/storage-index.js';
 import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
 import { Resource } from '@opentelemetry/resources';
+import { logger } from '@powersync/lib-services-framework';
 
 export interface MetricsOptions {
   disable_telemetry_sharing: boolean;
@@ -144,9 +144,9 @@ export class Metrics {
     if (Metrics.instance) {
       return;
     }
-    micro.logger.info('Configuring telemetry.');
+    logger.info('Configuring telemetry.');
 
-    micro.logger.info(
+    logger.info(
       `
 Attention:
 PowerSync collects completely anonymous telemetry regarding usage.
@@ -164,7 +164,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
     configuredExporters.push(prometheusExporter);
 
     if (!options.disable_telemetry_sharing) {
-      micro.logger.info('Sharing anonymous telemetry');
+      logger.info('Sharing anonymous telemetry');
       const periodicExporter = new PeriodicExportingMetricReader({
         exporter: new OTLPMetricExporter({
           url: options.internal_metrics_endpoint
@@ -189,7 +189,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
 
     Metrics.instance = new Metrics(meterProvider, prometheusExporter);
 
-    micro.logger.info('Telemetry configuration complete.');
+    logger.info('Telemetry configuration complete.');
   }
 
   public async shutdown(): Promise<void> {
@@ -212,7 +212,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
     function getMetrics() {
       if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
         cachedRequest = system.storage.getStorageMetrics().catch((e) => {
-          micro.logger.error(`Failed to get storage metrics`, e);
+          logger.error(`Failed to get storage metrics`, e);
           return null;
         });
         cacheTimestamp = Date.now();

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -1,6 +1,6 @@
-import * as mongo from '@/db/mongo.js';
-import * as storage from '@/storage/storage-index.js';
-import * as utils from '@/util/util-index.js';
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
 
 export const up = async (context?: utils.MigrationContext) => {
   const config = await utils.loadConfig(context?.runner_config);

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -1,6 +1,6 @@
-import * as mongo from '@/db/mongo.js';
-import * as storage from '@/storage/storage-index.js';
-import * as utils from '@/util/util-index.js';
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
 
 interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   /**

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -1,5 +1,5 @@
-import * as storage from '@/storage/storage-index.js';
-import * as utils from '@/util/util-index.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
 
 export const up = async (context?: utils.MigrationContext) => {
   const config = await utils.loadConfig(context?.runner_config);

package/src/migrations/definitions.ts

@@ -0,0 +1,21 @@
+export type Migration = {
+  name: string;
+  up: () => Promise<void>;
+  down: () => Promise<void>;
+};
+
+export enum Direction {
+  Up = 'up',
+  Down = 'down'
+}
+
+export type ExecutedMigration = {
+  name: string;
+  direction: Direction;
+  timestamp: Date;
+};
+
+export type MigrationState = {
+  last_run: string;
+  log: ExecutedMigration[];
+};

package/src/migrations/executor.ts

@@ -0,0 +1,87 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as defs from './definitions.js';
+import { MigrationStore } from './store/migration-store.js';
+
+type ExecuteParams = {
+  migrations: defs.Migration[];
+  state?: defs.MigrationState;
+
+  direction: defs.Direction;
+  count?: number;
+};
+
+export async function* execute(params: ExecuteParams): AsyncGenerator<defs.ExecutedMigration> {
+  let migrations = [...params.migrations];
+  if (params.direction === defs.Direction.Down) {
+    migrations = migrations.reverse();
+  }
+
+  let index = 0;
+
+  if (params.state) {
+    // Find the index of the last run
+    index = migrations.findIndex((migration) => {
+      return migration.name === params.state!.last_run;
+    });
+
+    if (index === -1) {
+      throw new Error(`The last run migration ${params.state?.last_run} was not found in the given set of migrations`);
+    }
+
+    // If we are migrating down then we want to include the last run migration, otherwise we want to start at the next one
+    if (params.direction === defs.Direction.Up) {
+      index += 1;
+    }
+  }
+
+  migrations = migrations.slice(index);
+
+  let i = 0;
+  for (const migration of migrations) {
+    if (params.count && params.count === i) {
+      return;
+    }
+
+    logger.info(`Executing ${migration.name} (${params.direction})`);
+    try {
+      switch (params.direction) {
+        case defs.Direction.Up: {
+          await migration.up();
+          break;
+        }
+        case defs.Direction.Down: {
+          await migration.down();
+          break;
+        }
+      }
+      logger.debug(`Success`);
+    } catch (err) {
+      logger.error(`Failed`, err);
+      process.exit(1);
+    }
+
+    yield {
+      name: migration.name,
+      direction: params.direction,
+      timestamp: new Date()
+    };
+
+    i++;
+  }
+}
+
+type WriteLogsParams = {
+  store: MigrationStore;
+  state?: defs.MigrationState;
+  log_stream: Iterable<defs.ExecutedMigration> | AsyncIterable<defs.ExecutedMigration>;
+};
+export const writeLogsToStore = async (params: WriteLogsParams): Promise<void> => {
+  const log = [...(params.state?.log || [])];
+  for await (const migration of params.log_stream) {
+    log.push(migration);
+    await params.store.save({
+      last_run: migration.name,
+      log: log
+    });
+  }
+};

package/src/migrations/migrations-index.ts

@@ -0,0 +1,3 @@
+export * from './definitions.js';
+export * from './executor.js';
+export * from './migrations.js';

package/src/migrations/migrations.ts

@@ -1,12 +1,13 @@
-import { locks } from '@journeyapps-platform/micro';
 import * as fs from 'fs/promises';
 import * as path from 'path';
 import { fileURLToPath } from 'url';
 
-import { Direction, createMongoMigrationStore, execute, writeLogsToStore } from '@journeyapps-platform/micro-migrate';
-
-import * as db from '@/db/db-index.js';
-import * as util from '@/util/util-index.js';
+import * as db from '../db/db-index.js';
+import * as util from '../util/util-index.js';
+import * as locks from '../locks/locks-index.js';
+import { Direction } from './definitions.js';
+import { createMongoMigrationStore } from './store/migration-store.js';
+import { execute, writeLogsToStore } from './executor.js';
 
 const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
 const MONGO_LOCK_PROCESS = 'migrations';
@@ -94,12 +95,7 @@ export const migrate = async (options: MigrationOptions) => {
     const migrations = await loadMigrations(MIGRATIONS_DIR, runner_config);
 
     // Use the provided config to connect to Mongo
-    const store = createMongoMigrationStore({
-      uri: storage.uri,
-      database: storage.database,
-      username: storage.username,
-      password: storage.password
-    });
+    const store = createMongoMigrationStore(clientDB);
 
     const state = await store.load();
 

package/src/migrations/store/migration-store.ts

@@ -0,0 +1,63 @@
+import { Db } from 'mongodb';
+import * as path from 'path';
+import * as defs from '../definitions.js';
+
+export type MigrationStore = {
+  load: () => Promise<defs.MigrationState | undefined>;
+  save: (state: defs.MigrationState) => Promise<void>;
+};
+
+/**
+ * A custom store for node-migrate which is used to save and load migrations that have
+ * been operated on to mongo.
+ */
+export const createMongoMigrationStore = (db: Db): MigrationStore => {
+  const collection = db.collection<defs.MigrationState>('migrations');
+
+  return {
+    load: async () => {
+      const state_entry = await collection.findOne();
+      if (!state_entry) {
+        return;
+      }
+
+      const { _id, ...state } = state_entry;
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to save
+       * state as `lastRun`.
+       */
+      let last_run = state.last_run;
+      if ('lastRun' in state) {
+        last_run = (state as any).lastRun;
+      }
+
+      /**
+       * This is for backwards compatibility. A previous version of the migration tool used to include the
+       * file extension in migration names. This strips that extension off if it exists
+       */
+      const extension = path.extname(last_run);
+      if (extension) {
+        last_run = last_run.replace(extension, '');
+      }
+
+      return {
+        last_run,
+        log: state.log || []
+      };
+    },
+
+    save: async (state: defs.MigrationState) => {
+      await collection.replaceOne(
+        {},
+        {
+          last_run: state.last_run,
+          log: state.log
+        },
+        {
+          upsert: true
+        }
+      );
+    }
+  };
+};

package/src/replication/WalConnection.ts

@@ -3,8 +3,8 @@ import { pgwireRows } from '@powersync/service-jpgwire';
 import { DEFAULT_TAG, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
 import { ReplicationError, TableInfo } from '@powersync/service-types';
 
-import * as storage from '@/storage/storage-index.js';
-import * as util from '@/util/util-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 import { ReplicaIdentityResult, getReplicationIdentityColumns } from './util.js';
 /**

package/src/replication/WalStream.ts

@@ -1,22 +1,19 @@
-import * as fs from 'fs/promises';
 import * as pgwire from '@powersync/service-jpgwire';
-import * as micro from '@journeyapps-platform/micro';
-import { logger } from '@journeyapps-platform/micro';
+import { container, errors, logger } from '@powersync/lib-services-framework';
 import { SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules';
 
-import * as storage from '@/storage/storage-index.js';
-import * as util from '@/util/util-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 import { getPgOutputRelation, getRelId, PgRelation } from './PgRelation.js';
 import { getReplicationIdentityColumns } from './util.js';
 import { WalConnection } from './WalConnection.js';
-import { Metrics } from '@/metrics/Metrics.js';
+import { Metrics } from '../metrics/Metrics.js';
 
 export const ZERO_LSN = '00000000/00000000';
 
 export interface WalStreamOptions {
   connections: util.PgManager;
-
   factory: storage.BucketStorageFactory;
   storage: storage.SyncRulesBucketStorage;
   abort_signal: AbortSignal;
@@ -160,7 +157,7 @@ export class WalStream {
         ]
       });
       if (rs.rows.length == 0) {
-        micro.logger.info(`Skipping ${tablePattern.schema}.${name} - not part of ${this.publication_name} publication`);
+        logger.info(`Skipping ${tablePattern.schema}.${name} - not part of ${this.publication_name} publication`);
         continue;
       }
 
@@ -190,7 +187,7 @@ export class WalStream {
 
     const status = await this.storage.getStatus();
     if (status.snapshot_done && status.checkpoint_lsn) {
-      micro.logger.info(`${slotName} Initial replication already done`);
+      logger.info(`${slotName} Initial replication already done`);
 
       let last_error = null;
 
@@ -199,8 +196,8 @@ export class WalStream {
         await touch();
 
         if (i == 0) {
-          util.captureException(last_error, {
-            level: micro.errors.ErrorSeverity.ERROR,
+          container.reporter.captureException(last_error, {
+            level: errors.ErrorSeverity.ERROR,
             metadata: {
               replication_slot: slotName
             }
@@ -222,11 +219,11 @@ export class WalStream {
             ]
           });
           // Success
-          micro.logger.info(`Slot ${slotName} appears healthy`);
+          logger.info(`Slot ${slotName} appears healthy`);
           return { needsInitialSync: false };
         } catch (e) {
           last_error = e;
-          micro.logger.warn(`${slotName} Replication slot error`, e);
+          logger.warn(`${slotName} Replication slot error`, e);
 
           if (this.stopped) {
             throw e;
@@ -240,8 +237,8 @@ export class WalStream {
             /replication slot.*does not exist/.test(e.message) ||
             /publication.*does not exist/.test(e.message)
           ) {
-            util.captureException(e, {
-              level: micro.errors.ErrorSeverity.WARNING,
+            container.reporter.captureException(e, {
+              level: errors.ErrorSeverity.WARNING,
               metadata: {
                 try_index: i,
                 replication_slot: slotName
@@ -253,7 +250,7 @@ export class WalStream {
             // Sample: publication "powersync" does not exist
             //   Happens when publication deleted or never created.
             //   Slot must be re-created in this case.
-            micro.logger.info(`${slotName} does not exist anymore, will create new slot`);
+            logger.info(`${slotName} does not exist anymore, will create new slot`);
 
             throw new MissingReplicationSlotError(`Replication slot ${slotName} does not exist anymore`);
           }
@@ -316,7 +313,7 @@ WHERE  oid = $1::regclass`,
     // with streaming replication.
     const lsn = pgwire.lsnMakeComparable(row[1]);
     const snapshot = row[2];
-    micro.logger.info(`Created replication slot ${slotName} at ${lsn} with snapshot ${snapshot}`);
+    logger.info(`Created replication slot ${slotName} at ${lsn} with snapshot ${snapshot}`);
 
     // https://stackoverflow.com/questions/70160769/postgres-logical-replication-starting-from-given-lsn
     await db.query('BEGIN');
@@ -338,9 +335,9 @@ WHERE  oid = $1::regclass`,
       // On Supabase, the default is 2 minutes.
       await db.query(`set local statement_timeout = 0`);
 
-      micro.logger.info(`${slotName} Starting initial replication`);
+      logger.info(`${slotName} Starting initial replication`);
       await this.initialReplication(db, lsn);
-      micro.logger.info(`${slotName} Initial replication done`);
+      logger.info(`${slotName} Initial replication done`);
       await db.query('COMMIT');
     } catch (e) {
       await db.query('ROLLBACK');
@@ -371,7 +368,7 @@ WHERE  oid = $1::regclass`,
   }
 
   private async snapshotTable(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, table: storage.SourceTable) {
-    micro.logger.info(`${this.slot_name} Replicating ${table.qualifiedName}`);
+    logger.info(`${this.slot_name} Replicating ${table.qualifiedName}`);
     const estimatedCount = await this.estimatedCount(db, table);
     let at = 0;
     let lastLogIndex = 0;
@@ -397,7 +394,7 @@ WHERE  oid = $1::regclass`,
         return q;
       });
       if (rows.length > 0 && at - lastLogIndex >= 5000) {
-        micro.logger.info(`${this.slot_name} Replicating ${table.qualifiedName} ${at}/${estimatedCount}`);
+        logger.info(`${this.slot_name} Replicating ${table.qualifiedName} ${at}/${estimatedCount}`);
         lastLogIndex = at;
       }
       if (this.abort_signal.aborted) {
@@ -586,7 +583,7 @@ WHERE  oid = $1::regclass`,
             await this.ack(msg.lsn!, replicationStream);
           } else {
             if (count % 100 == 0) {
-              micro.logger.info(`${this.slot_name} replicating op ${count} ${msg.lsn}`);
+              logger.info(`${this.slot_name} replicating op ${count} ${msg.lsn}`);
             }
 
             count += 1;
@@ -619,11 +616,9 @@ WHERE  oid = $1::regclass`,
   }
 }
 
-export async function touch() {
-  // FIXME: The probe does not actually check the timestamp on this.
-  // FIXME: We need a timeout of around 5+ minutes if we do start checking the timestamp,
-  // or reduce PING_INTERVAL.
-  await micro.signals.getSystemProbe().touch();
-  // FIXME: The above probe touches the wrong file
-  await fs.writeFile('.probes/poll', `${Date.now()}`);
+async function touch() {
+  // FIXME: The hosted Kubernetes probe does not actually check the timestamp on this.
+  // FIXME: We need a timeout of around 5+ minutes in Kubernetes if we do start checking the timestamp,
+  // or reduce PING_INTERVAL here.
+  return container.probes.touch();
 }