@powersync/service-core 0.0.0-dev-20240725112650 → 0.0.0-dev-20240918092408

package/src/api/schema.ts

@@ -1,99 +1,23 @@
-import type * as pgwire from '@powersync/service-jpgwire';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { DatabaseSchema, internal_routes } from '@powersync/service-types';
+import { internal_routes } from '@powersync/service-types';
 
-import * as util from '../util/util-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import * as api from '../api/api-index.js';
 
-export async function getConnectionsSchema(system: CorePowerSyncSystem): Promise<internal_routes.GetSchemaResponse> {
-  if (system.config.connection == null) {
-    return { connections: [] };
+export async function getConnectionsSchema(api: api.RouteAPI): Promise<internal_routes.GetSchemaResponse> {
+  if (!api) {
+    return {
+      connections: []
+    };
   }
-  const schemas = await getConnectionSchema(system.requirePgPool());
+
+  const baseConfig = await api.getSourceConfig();
+
   return {
     connections: [
       {
-        schemas,
-        tag: system.config.connection!.tag,
-        id: system.config.connection!.id
+        schemas: await api.getConnectionSchema(),
+        tag: baseConfig.tag!,
+        id: baseConfig.id
       }
     ]
   };
 }
-
-export async function getConnectionSchema(db: pgwire.PgClient): Promise<DatabaseSchema[]> {
-  // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
-  const results = await util.retriedQuery(
-    db,
-    `SELECT
-  tbl.schemaname,
-  tbl.tablename,
-  tbl.quoted_name,
-  json_agg(a ORDER BY attnum) as columns
-FROM
-  (
-    SELECT
-      n.nspname as schemaname,
-      c.relname as tablename,
-      (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) as quoted_name
-    FROM
-      pg_catalog.pg_class c
-      JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-    WHERE
-      c.relkind = 'r'
-      AND n.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
-      AND n.nspname not like 'pg_temp_%'
-      AND n.nspname not like 'pg_toast_temp_%'
-      AND c.relnatts > 0
-      AND has_schema_privilege(n.oid, 'USAGE') = true
-      AND has_table_privilege(quote_ident(n.nspname) || '.' || quote_ident(c.relname), 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER') = true
-  ) as tbl
-  LEFT JOIN (
-    SELECT
-      attrelid,
-      attname,
-      format_type(atttypid, atttypmod) as data_type,
-      (SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type,
-      attnum,
-      attisdropped
-    FROM
-      pg_attribute
-  ) as a ON (
-    a.attrelid = tbl.quoted_name::regclass
-    AND a.attnum > 0
-    AND NOT a.attisdropped
-    AND has_column_privilege(tbl.quoted_name, a.attname, 'SELECT, INSERT, UPDATE, REFERENCES')
-  )
-GROUP BY schemaname, tablename, quoted_name`
-  );
-  const rows = pgwireRows(results);
-
-  let schemas: Record<string, any> = {};
-
-  for (let row of rows) {
-    const schema = (schemas[row.schemaname] ??= {
-      name: row.schemaname,
-      tables: []
-    });
-    const table = {
-      name: row.tablename,
-      columns: [] as any[]
-    };
-    schema.tables.push(table);
-
-    const columnInfo = JSON.parse(row.columns);
-    for (let column of columnInfo) {
-      let pg_type = column.pg_type as string;
-      if (pg_type.startsWith('_')) {
-        pg_type = `${pg_type.substring(1)}[]`;
-      }
-      table.columns.push({
-        name: column.attname,
-        type: column.data_type,
-        pg_type: pg_type
-      });
-    }
-  }
-
-  return Object.values(schemas);
-}

package/src/auth/KeyStore.ts

@@ -1,9 +1,9 @@
+import { logger } from '@powersync/lib-services-framework';
 import * as jose from 'jose';
 import secs from '../util/secs.js';
-import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
-import { KeyCollector } from './KeyCollector.js';
 import { JwtPayload } from './JwtPayload.js';
-import { logger } from '@powersync/lib-services-framework';
+import { KeyCollector } from './KeyCollector.js';
+import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
 
 /**
  * KeyStore to get keys and verify tokens.
@@ -32,10 +32,13 @@ import { logger } from '@powersync/lib-services-framework';
  * If we have a matching kid, we can generally get a detailed error (e.g. signature verification failed, invalid algorithm, etc).
  * If we don't have a matching kid, we'll generally just get an error "Could not find an appropriate key...".
  */
-export class KeyStore {
-  private collector: KeyCollector;
+export class KeyStore<Collector extends KeyCollector = KeyCollector> {
+  /**
+   * @internal
+   */
+  collector: Collector;
 
-  constructor(collector: KeyCollector) {
+  constructor(collector: Collector) {
     this.collector = collector;
   }
 

package/src/auth/auth-index.ts

@@ -7,4 +7,3 @@ export * from './KeyStore.js';
 export * from './LeakyBucket.js';
 export * from './RemoteJWKSCollector.js';
 export * from './StaticKeyCollector.js';
-export * from './SupabaseKeyCollector.js';

package/src/entry/cli-entry.ts

@@ -1,10 +1,11 @@
 import { Command } from 'commander';
 
+import { logger } from '@powersync/lib-services-framework';
 import * as utils from '../util/util-index.js';
+import { registerCompactAction } from './commands/compact-action.js';
 import { registerMigrationAction } from './commands/migrate-action.js';
+import { registerStartAction } from './commands/start-action.js';
 import { registerTearDownAction } from './commands/teardown-action.js';
-import { registerCompactAction, registerStartAction } from './entry-index.js';
-import { logger } from '@powersync/lib-services-framework';
 
 /**
  * Generates a Commander program which serves as the entry point

package/src/entry/commands/compact-action.ts

@@ -2,8 +2,8 @@ import { Command } from 'commander';
 
 import { logger } from '@powersync/lib-services-framework';
 import * as v8 from 'v8';
-import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js';
-import { loadConfig } from '../../util/config.js';
+import * as storage from '../../storage/storage-index.js';
+import * as utils from '../../util/util-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'compact';
@@ -26,23 +26,26 @@ export function registerCompactAction(program: Command) {
   wrapConfigCommand(compactCommand);
 
   return compactCommand.description('Compact storage').action(async (options) => {
+    logger.info('Compacting storage...');
     const runnerConfig = extractRunnerOptions(options);
-
-    const config = await loadConfig(runnerConfig);
-    const { storage } = config;
-    const psdb = createPowerSyncMongo(storage);
+    const configuration = await utils.loadConfig(runnerConfig);
+    logger.info('Successfully loaded configuration...');
+    const { storage: storageConfig } = configuration;
+    logger.info('Connecting to storage...');
+    const psdb = storage.createPowerSyncMongo(storageConfig);
     const client = psdb.client;
     await client.connect();
     try {
-      const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
-      const active = await bucketStorage.getActiveSyncRules();
+      const bucketStorage = new storage.MongoBucketStorage(psdb, { slot_name_prefix: configuration.slot_name_prefix });
+      const active = await bucketStorage.getActiveSyncRulesContent();
       if (active == null) {
         logger.info('No active instance to compact');
         return;
       }
       const p = bucketStorage.getInstance(active);
+      logger.info('Performing compaction...');
       await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
-      logger.info('done');
+      logger.info('Successfully compacted storage.');
     } catch (e) {
       logger.error(`Failed to compact: ${e.toString()}`);
       process.exit(1);

package/src/entry/commands/migrate-action.ts

@@ -1,9 +1,8 @@
+import { logger } from '@powersync/lib-services-framework';
 import { Command } from 'commander';
 
+import * as migrations from '../../migrations/migrations-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
-import { migrate } from '../../migrations/migrations.js';
-import { Direction } from '../../migrations/definitions.js';
-import { logger } from '@powersync/lib-services-framework';
 
 const COMMAND_NAME = 'migrate';
 
@@ -15,13 +14,11 @@ export function registerMigrationAction(program: Command) {
   return migrationCommand
     .description('Run migrations')
     .argument('<direction>', 'Migration direction. `up` or `down`')
-    .action(async (direction: Direction, options) => {
-      const runnerConfig = extractRunnerOptions(options);
-
+    .action(async (direction: migrations.Direction, options) => {
       try {
-        await migrate({
+        await migrations.migrate({
           direction,
-          runner_config: runnerConfig
+          runner_config: extractRunnerOptions(options)
         });
 
         process.exit(0);

package/src/entry/commands/teardown-action.ts

@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 
-import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 import { teardown } from '../../runner/teardown.js';
+import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'teardown';
 
@@ -12,7 +12,7 @@ export function registerTearDownAction(program: Command) {
 
   return teardownCommand
     .argument('[ack]', 'Type `TEARDOWN` to confirm teardown should occur')
-    .description('Terminate all replicating sync rules, deleting the replication slots')
+    .description('Terminate all replicating sync rules, clear remote configuration and remove all data')
     .action(async (ack, options) => {
       if (ack !== 'TEARDOWN') {
         throw new Error('TEARDOWN was not acknowledged.');

package/src/index.ts

@@ -18,8 +18,11 @@ export * as framework from '@powersync/lib-services-framework';
 export * from './metrics/Metrics.js';
 export * as metrics from './metrics/Metrics.js';
 
-export * from './migrations/migrations.js';
 export * as migrations from './migrations/migrations-index.js';
+export * from './migrations/migrations.js';
+
+export * from './modules/modules-index.js';
+export * as modules from './modules/modules-index.js';
 
 export * from './replication/replication-index.js';
 export * as replication from './replication/replication-index.js';
@@ -33,7 +36,7 @@ export * as storage from './storage/storage-index.js';
 export * from './sync/sync-index.js';
 export * as sync from './sync/sync-index.js';
 
-export * from './system/CorePowerSyncSystem.js';
+export * from './system/system-index.js';
 export * as system from './system/system-index.js';
 
 export * from './util/util-index.js';

package/src/metrics/Metrics.ts

@@ -1,13 +1,11 @@
 import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
-import * as jpgwire from '@powersync/service-jpgwire';
-import * as util from '../util/util-index.js';
-import * as storage from '../storage/storage-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { Resource } from '@opentelemetry/resources';
+import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { logger } from '@powersync/lib-services-framework';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 export interface MetricsOptions {
   disable_telemetry_sharing: boolean;
@@ -202,7 +200,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
     this.concurrent_connections.add(0);
   }
 
-  public configureReplicationMetrics(system: CorePowerSyncSystem) {
+  public configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory) {
     // Rate limit collection of these stats, since it may be an expensive query
     const MINIMUM_INTERVAL = 60_000;
 
@@ -211,7 +209,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
 
     function getMetrics() {
       if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
-        cachedRequest = system.storage.getStorageMetrics().catch((e) => {
+        cachedRequest = bucketStorage.getStorageMetrics().catch((e) => {
           logger.error(`Failed to get storage metrics`, e);
           return null;
         });
@@ -240,14 +238,6 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
         result.observe(metrics.replication_size_bytes);
       }
     });
-
-    const class_scoped_data_replicated_bytes = this.data_replicated_bytes;
-    // Record replicated bytes using global jpgwire metrics.
-    jpgwire.setMetricsRecorder({
-      addBytesRead(bytes) {
-        class_scoped_data_replicated_bytes.add(bytes);
-      }
-    });
   }
 
   public async getMetricValueForTests(name: string): Promise<number | undefined> {

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -2,8 +2,11 @@ import * as mongo from '../../../db/mongo.js';
 import * as storage from '../../../storage/storage-index.js';
 import * as utils from '../../../util/util-index.js';
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+
+  const config = await utils.loadConfig(runner_config);
+
   const database = storage.createPowerSyncMongo(config.storage);
   await mongo.waitForAuth(database.db);
   try {
@@ -20,8 +23,10 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+
   const database = storage.createPowerSyncMongo(config.storage);
   try {
     if (await database.bucket_parameters.indexExists('lookup')) {

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -23,9 +23,11 @@ interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   auto_activate?: boolean;
 }
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
   const db = storage.createPowerSyncMongo(config.storage);
+
   await mongo.waitForAuth(db.db);
   try {
     // We keep the old flags for existing deployments still shutting down.
@@ -68,8 +70,9 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
 
   const db = storage.createPowerSyncMongo(config.storage);
   try {

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -1,8 +1,9 @@
 import * as storage from '../../../storage/storage-index.js';
 import * as utils from '../../../util/util-index.js';
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
   const db = storage.createPowerSyncMongo(config.storage);
 
   try {
@@ -17,8 +18,9 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
 
   const db = storage.createPowerSyncMongo(config.storage);
 

package/src/migrations/migrations.ts

@@ -2,13 +2,13 @@ import * as fs from 'fs/promises';
 import * as path from 'path';
 import { fileURLToPath } from 'url';
 
+import { logger } from '@powersync/lib-services-framework';
 import * as db from '../db/db-index.js';
-import * as util from '../util/util-index.js';
 import * as locks from '../locks/locks-index.js';
+import * as util from '../util/util-index.js';
 import { Direction } from './definitions.js';
-import { createMongoMigrationStore } from './store/migration-store.js';
 import { execute, writeLogsToStore } from './executor.js';
-import { logger } from '@powersync/lib-services-framework';
+import { createMongoMigrationStore } from './store/migration-store.js';
 
 const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
 const MONGO_LOCK_PROCESS = 'migrations';
@@ -23,18 +23,23 @@ export type MigrationOptions = {
   runner_config: util.RunnerConfig;
 };
 
+export type AutomaticMigrationParams = {
+  config: util.ResolvedPowerSyncConfig;
+  runner_config: util.RunnerConfig;
+};
+
 /**
  * Loads migrations and injects a custom context for loading the specified
  * runner configuration.
  */
-const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) => {
+const loadMigrations = async (dir: string, runnerConfig: util.RunnerConfig) => {
   const files = await fs.readdir(dir);
   const migrations = files.filter((file) => {
     return path.extname(file) === '.js';
   });
 
   const context: util.MigrationContext = {
-    runner_config
+    runner_config: runnerConfig
   };
 
   return await Promise.all(
@@ -55,14 +60,13 @@ const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) =>
 export const migrate = async (options: MigrationOptions) => {
   const { direction, runner_config } = options;
 
+  const config = await util.loadConfig(runner_config);
+  const { storage } = config;
   /**
    * Try and get Mongo from config file.
    * But this might not be available in Journey Micro as we use the standard Mongo.
    */
 
-  const config = await util.loadConfig(runner_config);
-  const { storage } = config;
-
   const client = db.mongo.createMongoClient(storage);
   logger.info('Connecting to MongoDB');
   await client.connect();
@@ -124,3 +128,15 @@ export const migrate = async (options: MigrationOptions) => {
     logger.info('Done with migrations');
   }
 };
+
+/**
+ * Ensures automatic migrations are executed
+ */
+export const ensureAutomaticMigrations = async (params: AutomaticMigrationParams) => {
+  if (!params.config.migrations?.disable_auto_migration) {
+    await migrate({
+      direction: Direction.Up,
+      runner_config: params.runner_config
+    });
+  }
+};

package/src/modules/AbstractModule.ts

@@ -0,0 +1,37 @@
+import { ServiceContextContainer } from '../system/ServiceContext.js';
+import { logger } from '@powersync/lib-services-framework';
+import winston from 'winston';
+import { PersistedSyncRulesContent } from '../storage/BucketStorage.js';
+
+export interface TearDownOptions {
+  /**
+   *  If required, tear down any configuration/state for the specific sync rules
+   */
+  syncRules?: PersistedSyncRulesContent[];
+}
+
+export interface AbstractModuleOptions {
+  name: string;
+}
+
+export abstract class AbstractModule {
+  protected logger: winston.Logger;
+
+  protected constructor(protected options: AbstractModuleOptions) {
+    this.logger = logger.child({ name: `Module:${options.name}` });
+  }
+
+  /**
+   *  Initialize the module using any required services from the ServiceContext
+   */
+  public abstract initialize(context: ServiceContextContainer): Promise<void>;
+
+  /**
+   *  Permanently clean up and dispose of any configuration or state for this module.
+   */
+  public abstract teardown(options: TearDownOptions): Promise<void>;
+
+  public get name() {
+    return this.options.name;
+  }
+}

package/src/modules/ModuleManager.ts

@@ -0,0 +1,34 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as system from '../system/system-index.js';
+import { AbstractModule, TearDownOptions } from './AbstractModule.js';
+/**
+ *  The module manager keeps track of activated modules
+ */
+export class ModuleManager {
+  private readonly modules: Map<string, AbstractModule> = new Map();
+
+  public register(modules: AbstractModule[]) {
+    for (const module of modules) {
+      if (this.modules.has(module.name)) {
+        logger.warn(`Module ${module.name} already registered, skipping.`);
+        continue;
+      }
+      this.modules.set(module.name, module);
+      logger.info(`Successfully registered Module ${module.name}.`);
+    }
+  }
+
+  async initialize(serviceContext: system.ServiceContextContainer) {
+    logger.info(`Initializing modules...`);
+    for (const module of this.modules.values()) {
+      await module.initialize(serviceContext);
+    }
+    logger.info(`Successfully Initialized modules.`);
+  }
+
+  async tearDown(options: TearDownOptions) {
+    for (const module of this.modules.values()) {
+      await module.teardown(options);
+    }
+  }
+}

package/src/modules/modules-index.ts

@@ -0,0 +1,2 @@
+export * from './ModuleManager.js';
+export * from './AbstractModule.js';

package/src/replication/AbstractReplicationJob.ts

@@ -0,0 +1,79 @@
+import * as storage from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+import { container, logger } from '@powersync/lib-services-framework';
+import winston from 'winston';
+
+export interface AbstractReplicationJobOptions {
+  id: string;
+  storage: storage.SyncRulesBucketStorage;
+  lock: storage.ReplicationLock;
+  rateLimiter: ErrorRateLimiter;
+}
+
+export abstract class AbstractReplicationJob {
+  protected logger: winston.Logger;
+  protected abortController = new AbortController();
+  protected isReplicatingPromise: Promise<void> | null = null;
+
+  protected constructor(protected options: AbstractReplicationJobOptions) {
+    this.logger = logger.child({ name: `ReplicationJob: ${options.id}` });
+  }
+
+  /**
+   *  Copy the initial data set from the data source if required and then keep it in sync.
+   */
+  abstract replicate(): Promise<void>;
+
+  /**
+   *  Ensure the connection to the data source remains active
+   */
+  abstract keepAlive(): Promise<void>;
+
+  /**
+   *  Start the replication process
+   */
+  public async start(): Promise<void> {
+    this.isReplicatingPromise = this.replicate()
+      .catch((ex) => {
+        container.reporter.captureException(ex, {
+          metadata: {
+            replicator: this.id
+          }
+        });
+        this.logger.error(`Replication failed.`, ex);
+      })
+      .finally(async () => {
+        this.abortController.abort();
+        await this.options.lock.release();
+      });
+  }
+
+  /**
+   *  Safely stop the replication process
+   */
+  public async stop(): Promise<void> {
+    this.logger.info(`Stopping ${this.id} replication job for sync rule iteration: ${this.storage.group_id}`);
+    this.abortController.abort();
+    await this.isReplicatingPromise;
+  }
+
+  public get id() {
+    return this.options.id;
+  }
+
+  public get storage() {
+    return this.options.storage;
+  }
+
+  protected get lock() {
+    return this.options.lock;
+  }
+
+  protected get rateLimiter() {
+    return this.options.rateLimiter;
+  }
+
+  public get isStopped(): boolean {
+    return this.abortController.signal.aborted;
+  }
+}