@powersync/service-core 0.8.8 → 0.10.0

package/src/entry/commands/compact-action.ts

@@ -2,8 +2,8 @@ import { Command } from 'commander';
 
 import { logger } from '@powersync/lib-services-framework';
 import * as v8 from 'v8';
-import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js';
-import { loadConfig } from '../../util/config.js';
+import * as storage from '../../storage/storage-index.js';
+import * as utils from '../../util/util-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'compact';
@@ -21,28 +21,40 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
 const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
 
 export function registerCompactAction(program: Command) {
-  const compactCommand = program.command(COMMAND_NAME);
+  const compactCommand = program
+    .command(COMMAND_NAME)
+    .option(`-b, --buckets [buckets]`, 'Bucket or bucket definition name (optional, comma-separate multiple names)');
 
   wrapConfigCommand(compactCommand);
 
   return compactCommand.description('Compact storage').action(async (options) => {
+    const buckets = options.buckets?.split(',');
+    if (buckets == null) {
+      logger.info('Compacting storage for all buckets...');
+    } else {
+      logger.info(`Compacting storage for ${buckets.join(', ')}...`);
+    }
     const runnerConfig = extractRunnerOptions(options);
-
-    const config = await loadConfig(runnerConfig);
-    const { storage } = config;
-    const psdb = createPowerSyncMongo(storage);
+    const configuration = await utils.loadConfig(runnerConfig);
+    logger.info('Successfully loaded configuration...');
+    const { storage: storageConfig } = configuration;
+    logger.info('Connecting to storage...');
+    const psdb = storage.createPowerSyncMongo(storageConfig);
     const client = psdb.client;
     await client.connect();
     try {
-      const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
-      const active = await bucketStorage.getActiveSyncRules();
+      const bucketStorage = new storage.MongoBucketStorage(psdb, {
+        slot_name_prefix: configuration.slot_name_prefix
+      });
+      const active = await bucketStorage.getActiveSyncRulesContent();
       if (active == null) {
         logger.info('No active instance to compact');
         return;
       }
-      const p = bucketStorage.getInstance(active);
-      await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
-      logger.info('done');
+      using p = bucketStorage.getInstance(active);
+      logger.info('Performing compaction...');
+      await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
+      logger.info('Successfully compacted storage.');
     } catch (e) {
       logger.error(`Failed to compact: ${e.toString()}`);
       process.exit(1);

package/src/entry/commands/migrate-action.ts

@@ -1,9 +1,8 @@
+import { logger } from '@powersync/lib-services-framework';
 import { Command } from 'commander';
 
+import * as migrations from '../../migrations/migrations-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
-import { migrate } from '../../migrations/migrations.js';
-import { Direction } from '../../migrations/definitions.js';
-import { logger } from '@powersync/lib-services-framework';
 
 const COMMAND_NAME = 'migrate';
 
@@ -15,13 +14,11 @@ export function registerMigrationAction(program: Command) {
   return migrationCommand
     .description('Run migrations')
     .argument('<direction>', 'Migration direction. `up` or `down`')
-    .action(async (direction: Direction, options) => {
-      const runnerConfig = extractRunnerOptions(options);
-
+    .action(async (direction: migrations.Direction, options) => {
       try {
-        await migrate({
+        await migrations.migrate({
           direction,
-          runner_config: runnerConfig
+          runner_config: extractRunnerOptions(options)
         });
 
         process.exit(0);

package/src/entry/commands/teardown-action.ts

@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 
-import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 import { teardown } from '../../runner/teardown.js';
+import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'teardown';
 
@@ -12,7 +12,7 @@ export function registerTearDownAction(program: Command) {
 
   return teardownCommand
     .argument('[ack]', 'Type `TEARDOWN` to confirm teardown should occur')
-    .description('Terminate all replicating sync rules, deleting the replication slots')
+    .description('Terminate all replicating sync rules, clear remote configuration and remove all data')
     .action(async (ack, options) => {
       if (ack !== 'TEARDOWN') {
         throw new Error('TEARDOWN was not acknowledged.');

package/src/index.ts

@@ -18,8 +18,11 @@ export * as framework from '@powersync/lib-services-framework';
 export * from './metrics/Metrics.js';
 export * as metrics from './metrics/Metrics.js';
 
-export * from './migrations/migrations.js';
 export * as migrations from './migrations/migrations-index.js';
+export * from './migrations/migrations.js';
+
+export * from './modules/modules-index.js';
+export * as modules from './modules/modules-index.js';
 
 export * from './replication/replication-index.js';
 export * as replication from './replication/replication-index.js';
@@ -33,7 +36,7 @@ export * as storage from './storage/storage-index.js';
 export * from './sync/sync-index.js';
 export * as sync from './sync/sync-index.js';
 
-export * from './system/CorePowerSyncSystem.js';
+export * from './system/system-index.js';
 export * as system from './system/system-index.js';
 
 export * from './util/util-index.js';

package/src/metrics/Metrics.ts

@@ -1,13 +1,11 @@
 import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
-import * as jpgwire from '@powersync/service-jpgwire';
-import * as util from '../util/util-index.js';
-import * as storage from '../storage/storage-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { Resource } from '@opentelemetry/resources';
+import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { logger } from '@powersync/lib-services-framework';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 export interface MetricsOptions {
   disable_telemetry_sharing: boolean;
@@ -202,7 +200,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
     this.concurrent_connections.add(0);
   }
 
-  public configureReplicationMetrics(system: CorePowerSyncSystem) {
+  public configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory) {
     // Rate limit collection of these stats, since it may be an expensive query
     const MINIMUM_INTERVAL = 60_000;
 
@@ -211,7 +209,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
 
     function getMetrics() {
       if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
-        cachedRequest = system.storage.getStorageMetrics().catch((e) => {
+        cachedRequest = bucketStorage.getStorageMetrics().catch((e) => {
           logger.error(`Failed to get storage metrics`, e);
           return null;
         });
@@ -240,14 +238,6 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
         result.observe(metrics.replication_size_bytes);
       }
     });
-
-    const class_scoped_data_replicated_bytes = this.data_replicated_bytes;
-    // Record replicated bytes using global jpgwire metrics.
-    jpgwire.setMetricsRecorder({
-      addBytesRead(bytes) {
-        class_scoped_data_replicated_bytes.add(bytes);
-      }
-    });
   }
 
   public async getMetricValueForTests(name: string): Promise<number | undefined> {

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -2,8 +2,11 @@ import * as mongo from '../../../db/mongo.js';
 import * as storage from '../../../storage/storage-index.js';
 import * as utils from '../../../util/util-index.js';
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+
+  const config = await utils.loadConfig(runner_config);
+
   const database = storage.createPowerSyncMongo(config.storage);
   await mongo.waitForAuth(database.db);
   try {
@@ -20,8 +23,10 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+
   const database = storage.createPowerSyncMongo(config.storage);
   try {
     if (await database.bucket_parameters.indexExists('lookup')) {

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -23,9 +23,11 @@ interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   auto_activate?: boolean;
 }
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
   const db = storage.createPowerSyncMongo(config.storage);
+
   await mongo.waitForAuth(db.db);
   try {
     // We keep the old flags for existing deployments still shutting down.
@@ -68,8 +70,9 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
 
   const db = storage.createPowerSyncMongo(config.storage);
   try {

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -1,8 +1,9 @@
 import * as storage from '../../../storage/storage-index.js';
 import * as utils from '../../../util/util-index.js';
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
   const db = storage.createPowerSyncMongo(config.storage);
 
   try {
@@ -17,8 +18,9 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
 
   const db = storage.createPowerSyncMongo(config.storage);
 

package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts

@@ -0,0 +1,37 @@
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
+
+const INDEX_NAME = 'user_sync_rule_unique';
+
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    await db.custom_write_checkpoints.createIndex(
+      {
+        user_id: 1,
+        sync_rules_id: 1
+      },
+      { name: INDEX_NAME, unique: true }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
+      await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/migrations.ts

@@ -2,13 +2,13 @@ import * as fs from 'fs/promises';
 import * as path from 'path';
 import { fileURLToPath } from 'url';
 
+import { logger } from '@powersync/lib-services-framework';
 import * as db from '../db/db-index.js';
-import * as util from '../util/util-index.js';
 import * as locks from '../locks/locks-index.js';
+import * as util from '../util/util-index.js';
 import { Direction } from './definitions.js';
-import { createMongoMigrationStore } from './store/migration-store.js';
 import { execute, writeLogsToStore } from './executor.js';
-import { logger } from '@powersync/lib-services-framework';
+import { createMongoMigrationStore } from './store/migration-store.js';
 
 const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
 const MONGO_LOCK_PROCESS = 'migrations';
@@ -23,18 +23,23 @@ export type MigrationOptions = {
   runner_config: util.RunnerConfig;
 };
 
+export type AutomaticMigrationParams = {
+  config: util.ResolvedPowerSyncConfig;
+  runner_config: util.RunnerConfig;
+};
+
 /**
  * Loads migrations and injects a custom context for loading the specified
  * runner configuration.
  */
-const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) => {
+const loadMigrations = async (dir: string, runnerConfig: util.RunnerConfig) => {
   const files = await fs.readdir(dir);
   const migrations = files.filter((file) => {
     return path.extname(file) === '.js';
   });
 
   const context: util.MigrationContext = {
-    runner_config
+    runner_config: runnerConfig
   };
 
   return await Promise.all(
@@ -55,14 +60,13 @@ const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) =>
 export const migrate = async (options: MigrationOptions) => {
   const { direction, runner_config } = options;
 
+  const config = await util.loadConfig(runner_config);
+  const { storage } = config;
   /**
    * Try and get Mongo from config file.
    * But this might not be available in Journey Micro as we use the standard Mongo.
    */
 
-  const config = await util.loadConfig(runner_config);
-  const { storage } = config;
-
   const client = db.mongo.createMongoClient(storage);
   logger.info('Connecting to MongoDB');
   await client.connect();
@@ -124,3 +128,15 @@ export const migrate = async (options: MigrationOptions) => {
     logger.info('Done with migrations');
   }
 };
+
+/**
+ * Ensures automatic migrations are executed
+ */
+export const ensureAutomaticMigrations = async (params: AutomaticMigrationParams) => {
+  if (!params.config.migrations?.disable_auto_migration) {
+    await migrate({
+      direction: Direction.Up,
+      runner_config: params.runner_config
+    });
+  }
+};

package/src/modules/AbstractModule.ts

@@ -0,0 +1,37 @@
+import { ServiceContextContainer } from '../system/ServiceContext.js';
+import { logger } from '@powersync/lib-services-framework';
+import winston from 'winston';
+import { PersistedSyncRulesContent } from '../storage/BucketStorage.js';
+
+export interface TearDownOptions {
+  /**
+   *  If required, tear down any configuration/state for the specific sync rules
+   */
+  syncRules?: PersistedSyncRulesContent[];
+}
+
+export interface AbstractModuleOptions {
+  name: string;
+}
+
+export abstract class AbstractModule {
+  protected logger: winston.Logger;
+
+  protected constructor(protected options: AbstractModuleOptions) {
+    this.logger = logger.child({ name: `Module:${options.name}` });
+  }
+
+  /**
+   *  Initialize the module using any required services from the ServiceContext
+   */
+  public abstract initialize(context: ServiceContextContainer): Promise<void>;
+
+  /**
+   *  Permanently clean up and dispose of any configuration or state for this module.
+   */
+  public abstract teardown(options: TearDownOptions): Promise<void>;
+
+  public get name() {
+    return this.options.name;
+  }
+}

package/src/modules/ModuleManager.ts

@@ -0,0 +1,34 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as system from '../system/system-index.js';
+import { AbstractModule, TearDownOptions } from './AbstractModule.js';
+/**
+ *  The module manager keeps track of activated modules
+ */
+export class ModuleManager {
+  private readonly modules: Map<string, AbstractModule> = new Map();
+
+  public register(modules: AbstractModule[]) {
+    for (const module of modules) {
+      if (this.modules.has(module.name)) {
+        logger.warn(`Module ${module.name} already registered, skipping.`);
+        continue;
+      }
+      this.modules.set(module.name, module);
+      logger.info(`Successfully registered Module ${module.name}.`);
+    }
+  }
+
+  async initialize(serviceContext: system.ServiceContextContainer) {
+    logger.info(`Initializing modules...`);
+    for (const module of this.modules.values()) {
+      await module.initialize(serviceContext);
+    }
+    logger.info(`Successfully Initialized modules.`);
+  }
+
+  async tearDown(options: TearDownOptions) {
+    for (const module of this.modules.values()) {
+      await module.teardown(options);
+    }
+  }
+}

package/src/modules/modules-index.ts

@@ -0,0 +1,2 @@
+export * from './ModuleManager.js';
+export * from './AbstractModule.js';

package/src/replication/AbstractReplicationJob.ts

@@ -0,0 +1,79 @@
+import { container, logger } from '@powersync/lib-services-framework';
+import winston from 'winston';
+import * as storage from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+
+export interface AbstractReplicationJobOptions {
+  id: string;
+  storage: storage.SyncRulesBucketStorage;
+  lock: storage.ReplicationLock;
+  rateLimiter: ErrorRateLimiter;
+}
+
+export abstract class AbstractReplicationJob {
+  protected logger: winston.Logger;
+  protected abortController = new AbortController();
+  protected isReplicatingPromise: Promise<void> | null = null;
+
+  protected constructor(protected options: AbstractReplicationJobOptions) {
+    this.logger = logger.child({ name: `ReplicationJob: ${this.id}` });
+  }
+
+  /**
+   *  Copy the initial data set from the data source if required and then keep it in sync.
+   */
+  abstract replicate(): Promise<void>;
+
+  /**
+   *  Ensure the connection to the data source remains active
+   */
+  abstract keepAlive(): Promise<void>;
+
+  /**
+   *  Start the replication process
+   */
+  public async start(): Promise<void> {
+    this.isReplicatingPromise = this.replicate()
+      .catch((ex) => {
+        container.reporter.captureException(ex, {
+          metadata: {
+            replicator: this.id
+          }
+        });
+        this.logger.error(`Replication failed.`, ex);
+      })
+      .finally(async () => {
+        this.abortController.abort();
+        await this.options.lock.release();
+      });
+  }
+
+  /**
+   *  Safely stop the replication process
+   */
+  public async stop(): Promise<void> {
+    this.logger.info(`Stopping replication job for sync rule iteration: ${this.storage.group_id}`);
+    this.abortController.abort();
+    await this.isReplicatingPromise;
+  }
+
+  public get id() {
+    return this.options.id;
+  }
+
+  public get storage() {
+    return this.options.storage;
+  }
+
+  protected get lock() {
+    return this.options.lock;
+  }
+
+  protected get rateLimiter() {
+    return this.options.rateLimiter;
+  }
+
+  public get isStopped(): boolean {
+    return this.abortController.signal.aborted;
+  }
+}