@powersync/service-core 0.0.2

package/src/index.ts

@@ -0,0 +1,37 @@
+// Provides global and namespaced exports
+
+export * from './api/api-index.js';
+export * as api from './api/api-index.js';
+
+export * from './auth/auth-index.js';
+export * as auth from './auth/auth-index.js';
+
+export * from './db/db-index.js';
+export * as db from './db/db-index.js';
+
+export * from './entry/entry-index.js';
+export * as entry from './entry/entry-index.js';
+
+export * from './metrics/metrics.js';
+export * as metrics from './metrics/metrics.js';
+
+export * from './migrations/migrations.js';
+export * as migrations from './migrations/migrations.js';
+
+export * from './replication/replication-index.js';
+export * as replication from './replication/replication-index.js';
+
+export * from './routes/routes-index.js';
+export * as routes from './routes/routes-index.js';
+
+export * from './storage/storage-index.js';
+export * as storage from './storage/storage-index.js';
+
+export * from './sync/sync-index.js';
+export * as sync from './sync/sync-index.js';
+
+export * from './system/CorePowerSyncSystem.js';
+export * as system from './system/CorePowerSyncSystem.js';
+
+export * from './util/util-index.js';
+export * as utils from './util/util-index.js';

package/src/metrics/metrics.ts

@@ -0,0 +1,169 @@
+import * as micro from '@journeyapps-platform/micro';
+import { ValueType } from '@opentelemetry/api';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
+import * as jpgwire from '@powersync/service-jpgwire';
+
+import * as util from '@/util/util-index.js';
+import * as storage from '@/storage/storage-index.js';
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+
+const meterProvider = new MeterProvider();
+
+const port: number = util.env.METRICS_PORT ?? 0;
+
+// Expose a pull-based metrics exporter on a prometheus endpoint.
+// This is not quite the standard approach of opentelemetry or journey-micro.
+// However, it avoids the need for running a separate sidecar that uses additional resources.
+// We can consider switching to the standard push-based architecture if needed later.
+
+export const exporter = new PrometheusExporter({ port: port, preventServerStart: true });
+meterProvider.addMetricReader(exporter);
+
+if (port > 0) {
+  exporter.startServer();
+}
+
+const meter = meterProvider.getMeter('powersync');
+
+// 1. Data processing / month
+
+// 1a. Postgres -> PowerSync
+// Record on replication pod
+export const data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', {
+  description: 'Uncompressed size of replicated data',
+  unit: 'bytes',
+  valueType: ValueType.INT
+});
+
+// 1b. PowerSync -> clients
+// Record on API pod
+export const data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', {
+  description: 'Uncompressed size of synced data',
+  unit: 'bytes',
+  valueType: ValueType.INT
+});
+
+// Unused for pricing
+// Record on replication pod
+export const rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', {
+  description: 'Total number of replicated rows',
+  valueType: ValueType.INT
+});
+
+// Unused for pricing
+// Record on replication pod
+export const transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', {
+  description: 'Total number of replicated transactions',
+  valueType: ValueType.INT
+});
+
+// Unused for pricing
+// Record on replication pod
+export const chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', {
+  description: 'Total number of replication chunks',
+  valueType: ValueType.INT
+});
+
+// 2. Sync operations / month
+// Record on API pod
+export const operations_synced_total = meter.createCounter('powersync_operations_synced_total', {
+  description: 'Number of operations synced',
+  valueType: ValueType.INT
+});
+
+// 3. Data hosted on PowerSync sync service
+// Record on replication pod
+
+// 3a. Replication storage -> raw data as received from Postgres.
+export const replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', {
+  description: 'Size of current data stored in PowerSync',
+  unit: 'bytes',
+  valueType: ValueType.INT
+});
+
+// 3b. Operations storage -> transformed history, as will be synced to clients
+export const operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', {
+  description: 'Size of operations stored in PowerSync',
+  unit: 'bytes',
+  valueType: ValueType.INT
+});
+
+// 3c. Parameter storage -> used for parameter queries
+export const parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', {
+  description: 'Size of parameter data stored in PowerSync',
+  unit: 'bytes',
+  valueType: ValueType.INT
+});
+
+// 4. Peak concurrent connections
+// Record on API pod
+
+export const concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', {
+  description: 'Number of concurrent sync connections',
+  valueType: ValueType.INT
+});
+
+export function configureApiMetrics() {
+  // Initialize the metric, so that it reports a value before connections
+  // have been opened.
+  concurrent_connections.add(0);
+}
+
+export function configureReplicationMetrics(system: CorePowerSyncSystem) {
+  // Rate limit collection of these stats, since it may be an expensive query
+  const MINIMUM_INTERVAL = 60_000;
+
+  let cachedRequest: Promise<storage.StorageMetrics | null> | undefined = undefined;
+  let cacheTimestamp = 0;
+
+  function getMetrics() {
+    if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
+      cachedRequest = system.storage.getStorageMetrics().catch((e) => {
+        micro.logger.error(`Failed to get storage metrics`, e);
+        return null;
+      });
+      cacheTimestamp = Date.now();
+    }
+    return cachedRequest;
+  }
+
+  operation_storage_size_bytes.addCallback(async (result) => {
+    const metrics = await getMetrics();
+    if (metrics) {
+      result.observe(metrics.operations_size_bytes);
+    }
+  });
+
+  parameter_storage_size_bytes.addCallback(async (result) => {
+    const metrics = await getMetrics();
+    if (metrics) {
+      result.observe(metrics.parameters_size_bytes);
+    }
+  });
+
+  replication_storage_size_bytes.addCallback(async (result) => {
+    const metrics = await getMetrics();
+    if (metrics) {
+      result.observe(metrics.replication_size_bytes);
+    }
+  });
+
+  // Record replicated bytes using global jpgwire metrics.
+  jpgwire.setMetricsRecorder({
+    addBytesRead(bytes) {
+      data_replicated_bytes.add(bytes);
+    }
+  });
+}
+
+export async function getMetricValueForTests(name: string): Promise<number | undefined> {
+  const metrics = await exporter.collect();
+  const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics;
+  const metric = scoped.find((metric) => metric.descriptor.name == name);
+  if (metric == null) {
+    throw new Error(`Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}`);
+  }
+  const point = metric.dataPoints[metric.dataPoints.length - 1];
+  return point?.value as number;
+}

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -0,0 +1,33 @@
+import * as mongo from '@/db/mongo.js';
+import * as storage from '@/storage/storage-index.js';
+import * as utils from '@/util/util-index.js';
+
+export const up = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const database = storage.createPowerSyncMongo(config.storage);
+  await mongo.waitForAuth(database.db);
+  try {
+    await database.bucket_parameters.createIndex(
+      {
+        'key.g': 1,
+        lookup: 1,
+        _id: 1
+      },
+      { name: 'lookup1' }
+    );
+  } finally {
+    await database.client.close();
+  }
+};
+
+export const down = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const database = storage.createPowerSyncMongo(config.storage);
+  try {
+    if (await database.bucket_parameters.indexExists('lookup')) {
+      await database.bucket_parameters.dropIndex('lookup1');
+    }
+  } finally {
+    await database.client.close();
+  }
+};

package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts

@@ -0,0 +1,5 @@
+export const up = async () => {
+  // No-op - we don't auto-create sync rules anymore
+};
+
+export const down = async () => {};

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -0,0 +1,99 @@
+import * as mongo from '@/db/mongo.js';
+import * as storage from '@/storage/storage-index.js';
+import * as utils from '@/util/util-index.js';
+
+interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
+  /**
+   * True if this is the active sync rules.
+   * requires `snapshot_done == true` and `replicating == true`.
+   */
+  active?: boolean;
+
+  /**
+   * True if this sync rules should be used for replication.
+   *
+   * During reprocessing, there is one sync rules with `replicating = true, active = true`,
+   * and one with `replicating = true, active = false, auto_activate = true`.
+   */
+  replicating?: boolean;
+
+  /**
+   * True if the sync rules should set `active = true` when `snapshot_done` = true.
+   */
+  auto_activate?: boolean;
+}
+
+export const up = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const db = storage.createPowerSyncMongo(config.storage);
+  await mongo.waitForAuth(db.db);
+  try {
+    // We keep the old flags for existing deployments still shutting down.
+
+    // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: true,
+        auto_activate: true
+      },
+      { $set: { state: storage.SyncRuleState.PROCESSING } }
+    );
+
+    // 2. Snapshot done: `active = true, snapshot_done = true, replicating = true, auto_activate = false`
+    await db.sync_rules.updateMany(
+      {
+        active: true
+      },
+      { $set: { state: storage.SyncRuleState.ACTIVE } }
+    );
+
+    // 3. Stopped: `active = false, snapshot_done = true, replicating = false, auto_activate = false`.
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: { $ne: true },
+        auto_activate: { $ne: true }
+      },
+      { $set: { state: storage.SyncRuleState.STOP } }
+    );
+
+    const remaining = await db.sync_rules.find({ state: null as any }).toArray();
+    if (remaining.length > 0) {
+      const slots = remaining.map((doc) => doc.slot_name).join(', ');
+      throw new Error(`Invalid state for sync rules: ${slots}`);
+    }
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+
+  const db = storage.createPowerSyncMongo(config.storage);
+  try {
+    await db.sync_rules.updateMany(
+      {
+        state: storage.SyncRuleState.ACTIVE
+      },
+      { $set: { active: true, replicating: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        state: storage.SyncRuleState.PROCESSING
+      },
+      { $set: { active: false, replicating: true, auto_activate: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        $or: [{ state: storage.SyncRuleState.STOP }, { state: storage.SyncRuleState.TERMINATED }]
+      },
+      { $set: { active: false, replicating: false, auto_activate: false } }
+    );
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -0,0 +1,32 @@
+import * as storage from '@/storage/storage-index.js';
+import * as utils from '@/util/util-index.js';
+
+export const up = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    await db.write_checkpoints.createIndex(
+      {
+        user_id: 1
+      },
+      { name: 'user_id' }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    if (await db.write_checkpoints.indexExists('user_id')) {
+      await db.write_checkpoints.dropIndex('user_id');
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/db/store.ts

@@ -0,0 +1,11 @@
+import * as micro_migrate from '@journeyapps-platform/micro-migrate';
+import * as utils from '@/util/util-index.js';
+
+const config = await utils.loadConfig();
+
+export default micro_migrate.createMongoMigrationStore({
+  uri: config.storage.uri,
+  database: config.storage.database,
+  username: config.storage.username,
+  password: config.storage.password
+});

package/src/migrations/migrations.ts

@@ -0,0 +1,122 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+import { Lock, createMongoLockManager } from '@journeyapps-platform/micro-locks';
+import { Direction, createMongoMigrationStore, execute, writeLogsToStore } from '@journeyapps-platform/micro-migrate';
+
+import * as db from '@/db/db-index.js';
+import * as util from '@/util/util-index.js';
+
+const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
+const MONGO_LOCK_PROCESS = 'migrations';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const MIGRATIONS_DIR = path.join(__dirname, '/db/migrations');
+
+export type MigrationOptions = {
+  direction: Direction;
+  runner_config: util.RunnerConfig;
+};
+
+/**
+ * Loads migrations and injects a custom context for loading the specified
+ * runner configuration.
+ */
+const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) => {
+  const files = await fs.readdir(dir);
+  const migrations = files.filter((file) => {
+    return path.extname(file) === '.js';
+  });
+
+  const context: util.MigrationContext = {
+    runner_config
+  };
+
+  return await Promise.all(
+    migrations.map(async (migration) => {
+      const module = await import(path.resolve(dir, migration));
+      return {
+        name: path.basename(migration).replace(path.extname(migration), ''),
+        up: () => module.up(context),
+        down: () => module.down(context)
+      };
+    })
+  );
+};
+
+/**
+ * Runs migration scripts exclusively using Mongo locks
+ */
+export const migrate = async (options: MigrationOptions) => {
+  const { direction, runner_config } = options;
+
+  /**
+   * Try and get Mongo from config file.
+   * But this might not be available in Journey Micro as we use the standard Mongo.
+   */
+
+  const config = await util.loadConfig(runner_config);
+  const { storage } = config;
+
+  const client = db.mongo.createMongoClient(storage);
+  await client.connect();
+
+  const clientDB = client.db(storage.database);
+  const collection = clientDB.collection<Lock>(DEFAULT_MONGO_LOCK_COLLECTION);
+
+  const manager = createMongoLockManager(collection, {
+    name: MONGO_LOCK_PROCESS
+  });
+
+  // Only one process should execute this at a time.
+  const lockId = await manager.acquire();
+
+  if (!lockId) {
+    throw new Error('Could not acquire lock_id');
+  }
+
+  let isReleased = false;
+  const releaseLock = async () => {
+    if (isReleased) {
+      return;
+    }
+    await manager.release(lockId);
+    isReleased = true;
+  };
+
+  // For the case where the migration is terminated
+  process.addListener('beforeExit', releaseLock);
+
+  try {
+    const migrations = await loadMigrations(MIGRATIONS_DIR, runner_config);
+
+    // Use the provided config to connect to Mongo
+    const store = createMongoMigrationStore({
+      uri: storage.uri,
+      database: storage.database,
+      username: storage.username,
+      password: storage.password
+    });
+
+    const state = await store.load();
+
+    const logStream = execute({
+      direction: direction,
+      migrations,
+      state
+    });
+
+    await writeLogsToStore({
+      log_stream: logStream,
+      store,
+      state
+    });
+  } finally {
+    await releaseLock();
+    await client.close(true);
+    process.removeListener('beforeExit', releaseLock);
+  }
+};

package/src/replication/ErrorRateLimiter.ts

@@ -0,0 +1,49 @@
+import { setTimeout } from 'timers/promises';
+
+export interface ErrorRateLimiter {
+  waitUntilAllowed(options?: { signal?: AbortSignal }): Promise<void>;
+  reportError(e: any): void;
+
+  mayPing(): boolean;
+}
+
+export class DefaultErrorRateLimiter implements ErrorRateLimiter {
+  nextAllowed: number = Date.now();
+
+  async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
+    const delay = Math.max(0, this.nextAllowed - Date.now());
+    this.setDelay(5_000);
+    await setTimeout(delay, undefined, { signal: options?.signal });
+  }
+
+  mayPing(): boolean {
+    return Date.now() >= this.nextAllowed;
+  }
+
+  reportError(e: any): void {
+    const message = (e.message as string) ?? '';
+    if (message.includes('password authentication failed')) {
+      // Wait 15 minutes, to avoid triggering Supabase's fail2ban
+      this.setDelay(900_000);
+    } else if (message.includes('ENOTFOUND')) {
+      // DNS lookup issue - incorrect URI or deleted instance
+      this.setDelay(120_000);
+    } else if (message.includes('ECONNREFUSED')) {
+      // Could be fail2ban or similar
+      this.setDelay(120_000);
+    } else if (
+      message.includes('Unable to do postgres query on ended pool') ||
+      message.includes('Postgres unexpectedly closed connection')
+    ) {
+      // Connection timed out - ignore / immediately retry
+      // We don't explicitly set the delay to 0, since there could have been another error that
+      // we need to respect.
+    } else {
+      this.setDelay(30_000);
+    }
+  }
+
+  private setDelay(delay: number) {
+    this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
+  }
+}

package/src/replication/PgRelation.ts

@@ -0,0 +1,42 @@
+import { PgoutputRelation } from '@powersync/service-jpgwire';
+
+export interface PgRelation {
+  readonly relationId: number;
+  readonly schema: string;
+  readonly name: string;
+  readonly replicaIdentity: ReplicationIdentity;
+  readonly replicationColumns: ReplicationColumn[];
+}
+
+export type ReplicationIdentity = 'default' | 'nothing' | 'full' | 'index';
+
+export interface ReplicationColumn {
+  readonly name: string;
+  readonly typeOid: number;
+}
+
+export function getReplicaIdColumns(relation: PgoutputRelation): ReplicationColumn[] {
+  if (relation.replicaIdentity == 'nothing') {
+    return [];
+  } else {
+    return relation.columns.filter((c) => (c.flags & 0b1) != 0).map((c) => ({ name: c.name, typeOid: c.typeOid }));
+  }
+}
+export function getRelId(source: PgoutputRelation): number {
+  // Source types are wrong here
+  const relId = (source as any).relationOid as number;
+  if (relId == null || typeof relId != 'number') {
+    throw new Error(`No relation id!`);
+  }
+  return relId;
+}
+
+export function getPgOutputRelation(source: PgoutputRelation): PgRelation {
+  return {
+    name: source.name,
+    schema: source.schema,
+    relationId: getRelId(source),
+    replicaIdentity: source.replicaIdentity,
+    replicationColumns: getReplicaIdColumns(source)
+  };
+}