@powersync/service-core 0.0.0-dev-20240620165206

package/src/index.ts

@@ -0,0 +1,40 @@
+// Provides global and namespaced exports
+
+export * from './api/api-index.js';
+export * as api from './api/api-index.js';
+
+export * from './auth/auth-index.js';
+export * as auth from './auth/auth-index.js';
+
+export * from './db/db-index.js';
+export * as db from './db/db-index.js';
+
+export * from './entry/entry-index.js';
+export * as entry from './entry/entry-index.js';
+
+// Re-export framework for easy use of Container API
+export * as framework from '@powersync/lib-services-framework';
+
+export * from './metrics/Metrics.js';
+export * as metrics from './metrics/Metrics.js';
+
+export * from './migrations/migrations.js';
+export * as migrations from './migrations/migrations-index.js';
+
+export * from './replication/replication-index.js';
+export * as replication from './replication/replication-index.js';
+
+export * from './routes/routes-index.js';
+export * as routes from './routes/routes-index.js';
+
+export * from './storage/storage-index.js';
+export * as storage from './storage/storage-index.js';
+
+export * from './sync/sync-index.js';
+export * as sync from './sync/sync-index.js';
+
+export * from './system/CorePowerSyncSystem.js';
+export * as system from './system/system-index.js';
+
+export * from './util/util-index.js';
+export * as utils from './util/util-index.js';

package/src/locks/LockManager.ts

@@ -0,0 +1,16 @@
+import * as bson from 'bson';
+
+export class LockActiveError extends Error {
+  constructor() {
+    super('Lock is already active');
+    this.name = this.constructor.name;
+  }
+}
+
+export type LockManager = {
+  acquire: () => Promise<bson.ObjectId | null>;
+  refresh: (lock_id: bson.ObjectId) => Promise<void>;
+  release: (lock_id: bson.ObjectId) => Promise<void>;
+
+  lock: (handler: (refresh: () => Promise<void>) => Promise<void>) => Promise<void>;
+};

package/src/locks/MongoLocks.ts

@@ -0,0 +1,142 @@
+import * as mongo from 'mongodb';
+import * as bson from 'bson';
+import { LockActiveError, LockManager } from './LockManager.js';
+
+/**
+ * Lock Document Schema
+ */
+export type Lock = {
+  name: string;
+  active_lock?: {
+    lock_id: bson.ObjectId;
+    ts: Date;
+  };
+};
+
+export type Collection = mongo.Collection<Lock>;
+
+export type AcquireLockParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL of the lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+const DEFAULT_LOCK_TIMEOUT = 60 * 1000; // 1 minute
+
+const acquireLock = async (collection: Collection, params: AcquireLockParams) => {
+  const now = new Date();
+  const lock_timeout = params.timeout ?? DEFAULT_LOCK_TIMEOUT;
+  const lock_id = new bson.ObjectId();
+
+  await collection.updateOne(
+    {
+      name: params.name
+    },
+    {
+      $setOnInsert: {
+        name: params.name
+      }
+    },
+    {
+      upsert: true
+    }
+  );
+
+  const expired_ts = now.getTime() - lock_timeout;
+
+  const res = await collection.updateOne(
+    {
+      $and: [
+        { name: params.name },
+        {
+          $or: [{ active_lock: { $exists: false } }, { 'active_lock.ts': { $lte: new Date(expired_ts) } }]
+        }
+      ]
+    },
+    {
+      $set: {
+        active_lock: {
+          lock_id: lock_id,
+          ts: now
+        }
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    return null;
+  }
+
+  return lock_id;
+};
+
+const refreshLock = async (collection: Collection, lock_id: bson.ObjectId) => {
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lock_id
+    },
+    {
+      $set: {
+        'active_lock.ts': new Date()
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not refresh');
+  }
+};
+
+export const releaseLock = async (collection: Collection, lock_id: bson.ObjectId) => {
+  const res = await collection.updateOne(
+    {
+      'active_lock.lock_id': lock_id
+    },
+    {
+      $unset: {
+        active_lock: true
+      }
+    }
+  );
+
+  if (res.modifiedCount === 0) {
+    throw new Error('Lock not found, could not release');
+  }
+};
+
+export type CreateLockManagerParams = {
+  /**
+   * Name of the process/user trying to acquire the lock.
+   */
+  name: string;
+  /**
+   * The TTL for each lock (ms). Default: 60000 ms (1 min)
+   */
+  timeout?: number;
+};
+
+export const createMongoLockManager = (collection: Collection, params: CreateLockManagerParams): LockManager => {
+  return {
+    acquire: () => acquireLock(collection, params),
+    refresh: (lock_id: bson.ObjectId) => refreshLock(collection, lock_id),
+    release: (lock_id: bson.ObjectId) => releaseLock(collection, lock_id),
+
+    lock: async (handler) => {
+      const lock_id = await acquireLock(collection, params);
+      if (!lock_id) {
+        throw new LockActiveError();
+      }
+
+      try {
+        await handler(() => refreshLock(collection, lock_id));
+      } finally {
+        await releaseLock(collection, lock_id);
+      }
+    }
+  };
+};

package/src/locks/locks-index.ts

@@ -0,0 +1,2 @@
+export * from './LockManager.js';
+export * from './MongoLocks.js';

package/src/metrics/Metrics.ts

@@ -0,0 +1,265 @@
+import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
+import * as jpgwire from '@powersync/service-jpgwire';
+import * as util from '../util/util-index.js';
+import * as storage from '../storage/storage-index.js';
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { Resource } from '@opentelemetry/resources';
+import { logger } from '@powersync/lib-services-framework';
+
+export interface MetricsOptions {
+  disable_telemetry_sharing: boolean;
+  powersync_instance_id: string;
+  internal_metrics_endpoint: string;
+}
+
+export class Metrics {
+  private static instance: Metrics;
+
+  private prometheusExporter: PrometheusExporter;
+  private meterProvider: MeterProvider;
+
+  // Metrics
+  // 1. Data processing / month
+
+  // 1a. Postgres -> PowerSync
+  // Record on replication pod
+  public data_replicated_bytes: Counter<Attributes>;
+  // 1b. PowerSync -> clients
+  // Record on API pod
+  public data_synced_bytes: Counter<Attributes>;
+  // Unused for pricing
+  // Record on replication pod
+  public rows_replicated_total: Counter<Attributes>;
+  // Unused for pricing
+  // Record on replication pod
+  public transactions_replicated_total: Counter<Attributes>;
+  // Unused for pricing
+  // Record on replication pod
+  public chunks_replicated_total: Counter<Attributes>;
+
+  // 2. Sync operations / month
+
+  // Record on API pod
+  public operations_synced_total: Counter<Attributes>;
+
+  // 3. Data hosted on PowerSync sync service
+
+  // Record on replication pod
+  // 3a. Replication storage -> raw data as received from Postgres.
+  public replication_storage_size_bytes: ObservableGauge<Attributes>;
+  // 3b. Operations storage -> transformed history, as will be synced to clients
+  public operation_storage_size_bytes: ObservableGauge<Attributes>;
+  // 3c. Parameter storage -> used for parameter queries
+  public parameter_storage_size_bytes: ObservableGauge<Attributes>;
+
+  // 4. Peak concurrent connections
+
+  // Record on API pod
+  public concurrent_connections: UpDownCounter<Attributes>;
+
+  private constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
+    this.meterProvider = meterProvider;
+    this.prometheusExporter = prometheusExporter;
+    const meter = meterProvider.getMeter('powersync');
+
+    this.data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', {
+      description: 'Uncompressed size of replicated data',
+      unit: 'bytes',
+      valueType: ValueType.INT
+    });
+
+    this.data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', {
+      description: 'Uncompressed size of synced data',
+      unit: 'bytes',
+      valueType: ValueType.INT
+    });
+
+    this.rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', {
+      description: 'Total number of replicated rows',
+      valueType: ValueType.INT
+    });
+
+    this.transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', {
+      description: 'Total number of replicated transactions',
+      valueType: ValueType.INT
+    });
+
+    this.chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', {
+      description: 'Total number of replication chunks',
+      valueType: ValueType.INT
+    });
+
+    this.operations_synced_total = meter.createCounter('powersync_operations_synced_total', {
+      description: 'Number of operations synced',
+      valueType: ValueType.INT
+    });
+
+    this.replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', {
+      description: 'Size of current data stored in PowerSync',
+      unit: 'bytes',
+      valueType: ValueType.INT
+    });
+
+    this.operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', {
+      description: 'Size of operations stored in PowerSync',
+      unit: 'bytes',
+      valueType: ValueType.INT
+    });
+
+    this.parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', {
+      description: 'Size of parameter data stored in PowerSync',
+      unit: 'bytes',
+      valueType: ValueType.INT
+    });
+
+    this.concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', {
+      description: 'Number of concurrent sync connections',
+      valueType: ValueType.INT
+    });
+  }
+
+  // Generally only useful for tests. Note: gauges are ignored here.
+  resetCounters() {
+    this.data_replicated_bytes.add(0);
+    this.data_synced_bytes.add(0);
+    this.rows_replicated_total.add(0);
+    this.transactions_replicated_total.add(0);
+    this.chunks_replicated_total.add(0);
+    this.operations_synced_total.add(0);
+    this.concurrent_connections.add(0);
+  }
+
+  public static getInstance(): Metrics {
+    if (!Metrics.instance) {
+      throw new Error('Metrics have not been initialised');
+    }
+
+    return Metrics.instance;
+  }
+
+  public static async initialise(options: MetricsOptions): Promise<void> {
+    if (Metrics.instance) {
+      return;
+    }
+    logger.info('Configuring telemetry.');
+
+    logger.info(
+      `
+Attention:
+PowerSync collects completely anonymous telemetry regarding usage.
+This information is used to shape our roadmap to better serve our customers.
+You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
+https://docs.powersync.com/self-hosting/telemetry
+Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'}
+    `.trim()
+    );
+
+    const configuredExporters: MetricReader[] = [];
+
+    const port: number = util.env.METRICS_PORT ?? 0;
+    const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true });
+    configuredExporters.push(prometheusExporter);
+
+    if (!options.disable_telemetry_sharing) {
+      logger.info('Sharing anonymous telemetry');
+      const periodicExporter = new PeriodicExportingMetricReader({
+        exporter: new OTLPMetricExporter({
+          url: options.internal_metrics_endpoint
+        }),
+        exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
+      });
+
+      configuredExporters.push(periodicExporter);
+    }
+
+    const meterProvider = new MeterProvider({
+      resource: new Resource({
+        ['service']: 'PowerSync',
+        ['instance_id']: options.powersync_instance_id
+      }),
+      readers: configuredExporters
+    });
+
+    if (port > 0) {
+      await prometheusExporter.startServer();
+    }
+
+    Metrics.instance = new Metrics(meterProvider, prometheusExporter);
+
+    logger.info('Telemetry configuration complete.');
+  }
+
+  public async shutdown(): Promise<void> {
+    await this.meterProvider.shutdown();
+  }
+
+  public configureApiMetrics() {
+    // Initialize the metric, so that it reports a value before connections
+    // have been opened.
+    this.concurrent_connections.add(0);
+  }
+
+  public configureReplicationMetrics(system: CorePowerSyncSystem) {
+    // Rate limit collection of these stats, since it may be an expensive query
+    const MINIMUM_INTERVAL = 60_000;
+
+    let cachedRequest: Promise<storage.StorageMetrics | null> | undefined = undefined;
+    let cacheTimestamp = 0;
+
+    function getMetrics() {
+      if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
+        cachedRequest = system.storage.getStorageMetrics().catch((e) => {
+          logger.error(`Failed to get storage metrics`, e);
+          return null;
+        });
+        cacheTimestamp = Date.now();
+      }
+      return cachedRequest;
+    }
+
+    this.operation_storage_size_bytes.addCallback(async (result) => {
+      const metrics = await getMetrics();
+      if (metrics) {
+        result.observe(metrics.operations_size_bytes);
+      }
+    });
+
+    this.parameter_storage_size_bytes.addCallback(async (result) => {
+      const metrics = await getMetrics();
+      if (metrics) {
+        result.observe(metrics.parameters_size_bytes);
+      }
+    });
+
+    this.replication_storage_size_bytes.addCallback(async (result) => {
+      const metrics = await getMetrics();
+      if (metrics) {
+        result.observe(metrics.replication_size_bytes);
+      }
+    });
+
+    const class_scoped_data_replicated_bytes = this.data_replicated_bytes;
+    // Record replicated bytes using global jpgwire metrics.
+    jpgwire.setMetricsRecorder({
+      addBytesRead(bytes) {
+        class_scoped_data_replicated_bytes.add(bytes);
+      }
+    });
+  }
+
+  public async getMetricValueForTests(name: string): Promise<number | undefined> {
+    const metrics = await this.prometheusExporter.collect();
+    const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics;
+    const metric = scoped.find((metric) => metric.descriptor.name == name);
+    if (metric == null) {
+      throw new Error(
+        `Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}`
+      );
+    }
+    const point = metric.dataPoints[metric.dataPoints.length - 1];
+    return point?.value as number;
+  }
+}

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -0,0 +1,33 @@
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
+
+export const up = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const database = storage.createPowerSyncMongo(config.storage);
+  await mongo.waitForAuth(database.db);
+  try {
+    await database.bucket_parameters.createIndex(
+      {
+        'key.g': 1,
+        lookup: 1,
+        _id: 1
+      },
+      { name: 'lookup1' }
+    );
+  } finally {
+    await database.client.close();
+  }
+};
+
+export const down = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const database = storage.createPowerSyncMongo(config.storage);
+  try {
+    if (await database.bucket_parameters.indexExists('lookup')) {
+      await database.bucket_parameters.dropIndex('lookup1');
+    }
+  } finally {
+    await database.client.close();
+  }
+};

package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts

@@ -0,0 +1,5 @@
+export const up = async () => {
+  // No-op - we don't auto-create sync rules anymore
+};
+
+export const down = async () => {};

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -0,0 +1,99 @@
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
+
+interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
+  /**
+   * True if this is the active sync rules.
+   * requires `snapshot_done == true` and `replicating == true`.
+   */
+  active?: boolean;
+
+  /**
+   * True if this sync rules should be used for replication.
+   *
+   * During reprocessing, there is one sync rules with `replicating = true, active = true`,
+   * and one with `replicating = true, active = false, auto_activate = true`.
+   */
+  replicating?: boolean;
+
+  /**
+   * True if the sync rules should set `active = true` when `snapshot_done` = true.
+   */
+  auto_activate?: boolean;
+}
+
+export const up = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const db = storage.createPowerSyncMongo(config.storage);
+  await mongo.waitForAuth(db.db);
+  try {
+    // We keep the old flags for existing deployments still shutting down.
+
+    // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: true,
+        auto_activate: true
+      },
+      { $set: { state: storage.SyncRuleState.PROCESSING } }
+    );
+
+    // 2. Snapshot done: `active = true, snapshot_done = true, replicating = true, auto_activate = false`
+    await db.sync_rules.updateMany(
+      {
+        active: true
+      },
+      { $set: { state: storage.SyncRuleState.ACTIVE } }
+    );
+
+    // 3. Stopped: `active = false, snapshot_done = true, replicating = false, auto_activate = false`.
+    await db.sync_rules.updateMany(
+      {
+        active: { $ne: true },
+        replicating: { $ne: true },
+        auto_activate: { $ne: true }
+      },
+      { $set: { state: storage.SyncRuleState.STOP } }
+    );
+
+    const remaining = await db.sync_rules.find({ state: null as any }).toArray();
+    if (remaining.length > 0) {
+      const slots = remaining.map((doc) => doc.slot_name).join(', ');
+      throw new Error(`Invalid state for sync rules: ${slots}`);
+    }
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+
+  const db = storage.createPowerSyncMongo(config.storage);
+  try {
+    await db.sync_rules.updateMany(
+      {
+        state: storage.SyncRuleState.ACTIVE
+      },
+      { $set: { active: true, replicating: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        state: storage.SyncRuleState.PROCESSING
+      },
+      { $set: { active: false, replicating: true, auto_activate: true } }
+    );
+
+    await db.sync_rules.updateMany(
+      {
+        $or: [{ state: storage.SyncRuleState.STOP }, { state: storage.SyncRuleState.TERMINATED }]
+      },
+      { $set: { active: false, replicating: false, auto_activate: false } }
+    );
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -0,0 +1,32 @@
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
+
+export const up = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    await db.write_checkpoints.createIndex(
+      {
+        user_id: 1
+      },
+      { name: 'user_id' }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down = async (context?: utils.MigrationContext) => {
+  const config = await utils.loadConfig(context?.runner_config);
+
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    if (await db.write_checkpoints.indexExists('user_id')) {
+      await db.write_checkpoints.dropIndex('user_id');
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/definitions.ts

@@ -0,0 +1,21 @@
+export type Migration = {
+  name: string;
+  up: () => Promise<void>;
+  down: () => Promise<void>;
+};
+
+export enum Direction {
+  Up = 'up',
+  Down = 'down'
+}
+
+export type ExecutedMigration = {
+  name: string;
+  direction: Direction;
+  timestamp: Date;
+};
+
+export type MigrationState = {
+  last_run: string;
+  log: ExecutedMigration[];
+};