@powersync/service-core 1.10.1 → 1.11.0

package/src/api/api-metrics.ts

@@ -0,0 +1,35 @@
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
+import { APIMetric } from '@powersync/service-types';
+
+/**
+ *  Create and register the core API metrics.
+ *  @param engine
+ */
+export function createCoreAPIMetrics(engine: MetricsEngine): void {
+  engine.createCounter({
+    name: APIMetric.DATA_SYNCED_BYTES,
+    description: 'Uncompressed size of synced data',
+    unit: 'bytes'
+  });
+
+  engine.createCounter({
+    name: APIMetric.OPERATIONS_SYNCED,
+    description: 'Number of operations synced'
+  });
+
+  engine.createUpDownCounter({
+    name: APIMetric.CONCURRENT_CONNECTIONS,
+    description: 'Number of concurrent sync connections'
+  });
+}
+
+/**
+ *  Initialise the core API metrics. This should be called after the metrics have been created.
+ *  @param engine
+ */
+export function initializeCoreAPIMetrics(engine: MetricsEngine): void {
+  const concurrent_connections = engine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS);
+
+  // Initialize the metric, so that it reports a value before connections have been opened.
+  concurrent_connections.add(0);
+}

package/src/index.ts

@@ -12,8 +12,8 @@ export * as entry from './entry/entry-index.js';
 // Re-export framework for easy use of Container API
 export * as framework from '@powersync/lib-services-framework';
 
-export * from './metrics/Metrics.js';
-export * as metrics from './metrics/Metrics.js';
+export * from './metrics/metrics-index.js';
+export * as metrics from './metrics/metrics-index.js';
 
 export * from './migrations/migrations-index.js';
 export * as migrations from './migrations/migrations-index.js';

package/src/metrics/MetricsEngine.ts

@@ -0,0 +1,98 @@
+import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { Counter, UpDownCounter, ObservableGauge, MetricMetadata, MetricsFactory } from './metrics-interfaces.js';
+
+export interface MetricsEngineOptions {
+  factory: MetricsFactory;
+  disable_telemetry_sharing: boolean;
+}
+
+export class MetricsEngine {
+  private counters: Map<string, Counter>;
+  private upDownCounters: Map<string, UpDownCounter>;
+  private observableGauges: Map<string, ObservableGauge>;
+
+  constructor(private options: MetricsEngineOptions) {
+    this.counters = new Map();
+    this.upDownCounters = new Map();
+    this.observableGauges = new Map();
+  }
+
+  private get factory(): MetricsFactory {
+    return this.options.factory;
+  }
+
+  createCounter(metadata: MetricMetadata): Counter {
+    if (this.counters.has(metadata.name)) {
+      logger.warn(`Counter with name ${metadata.name} already created and registered, skipping.`);
+      return this.counters.get(metadata.name)!;
+    }
+
+    const counter = this.factory.createCounter(metadata);
+    this.counters.set(metadata.name, counter);
+    return counter;
+  }
+  createUpDownCounter(metadata: MetricMetadata): UpDownCounter {
+    if (this.upDownCounters.has(metadata.name)) {
+      logger.warn(`UpDownCounter with name ${metadata.name} already created and registered, skipping.`);
+      return this.upDownCounters.get(metadata.name)!;
+    }
+
+    const upDownCounter = this.factory.createUpDownCounter(metadata);
+    this.upDownCounters.set(metadata.name, upDownCounter);
+    return upDownCounter;
+  }
+
+  createObservableGauge(metadata: MetricMetadata): ObservableGauge {
+    if (this.observableGauges.has(metadata.name)) {
+      logger.warn(`ObservableGauge with name ${metadata.name} already created and registered, skipping.`);
+      return this.observableGauges.get(metadata.name)!;
+    }
+
+    const observableGauge = this.factory.createObservableGauge(metadata);
+    this.observableGauges.set(metadata.name, observableGauge);
+    return observableGauge;
+  }
+
+  getCounter(name: string): Counter {
+    const counter = this.counters.get(name);
+    if (!counter) {
+      throw new ServiceAssertionError(`Counter '${name}' has not been created and registered yet.`);
+    }
+    return counter;
+  }
+
+  getUpDownCounter(name: string): UpDownCounter {
+    const upDownCounter = this.upDownCounters.get(name);
+    if (!upDownCounter) {
+      throw new ServiceAssertionError(`UpDownCounter '${name}' has not been created and registered yet.`);
+    }
+    return upDownCounter;
+  }
+
+  getObservableGauge(name: string): ObservableGauge {
+    const observableGauge = this.observableGauges.get(name);
+    if (!observableGauge) {
+      throw new ServiceAssertionError(`ObservableGauge '${name}' has not been created and registered yet.`);
+    }
+    return observableGauge;
+  }
+
+  public async start(): Promise<void> {
+    logger.info(
+      `
+Attention:
+PowerSync collects completely anonymous telemetry regarding usage.
+This information is used to shape our roadmap to better serve our customers.
+You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
+https://docs.powersync.com/self-hosting/lifecycle-maintenance/telemetry
+    `.trim()
+    );
+    logger.info(`Anonymous telemetry is currently: ${this.options.disable_telemetry_sharing ? 'disabled' : 'enabled'}`);
+
+    logger.info('Successfully started Metrics Engine.');
+  }
+
+  public async shutdown(): Promise<void> {
+    logger.info('Successfully shut down Metrics Engine.');
+  }
+}

package/src/metrics/metrics-index.ts

@@ -0,0 +1,5 @@
+export * from './MetricsEngine.js';
+export * from './metrics-interfaces.js';
+export * from './register-metrics.js';
+export * from './open-telemetry/OpenTelemetryMetricsFactory.js';
+export * from './open-telemetry/util.js';

package/src/metrics/metrics-interfaces.ts

@@ -0,0 +1,41 @@
+export interface Counter {
+  /**
+   *  Increment the counter by the given value. Only positive numbers are valid.
+   *  @param value
+   */
+  add(value: number): void;
+}
+
+export interface UpDownCounter {
+  /**
+   *  Increment or decrement(if negative) the counter by the given value.
+   *  @param value
+   */
+  add(value: number): void;
+}
+
+export interface ObservableGauge {
+  /**
+   *  Set a value provider that provides the value for the gauge at the time of observation.
+   *  @param valueProvider
+   */
+  setValueProvider(valueProvider: () => Promise<number | undefined>): void;
+}
+
+export enum Precision {
+  INT = 'int',
+  DOUBLE = 'double'
+}
+
+export interface MetricMetadata {
+  name: string;
+  description?: string;
+  unit?: string;
+  precision?: Precision;
+}
+
+export interface MetricsFactory {
+  createCounter(metadata: MetricMetadata): Counter;
+  createUpDownCounter(metadata: MetricMetadata): UpDownCounter;
+  createObservableGauge(metadata: MetricMetadata): ObservableGauge;
+}

package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts

@@ -0,0 +1,66 @@
+import { Meter, ValueType } from '@opentelemetry/api';
+import {
+  Counter,
+  ObservableGauge,
+  UpDownCounter,
+  MetricMetadata,
+  MetricsFactory,
+  Precision
+} from '../metrics-interfaces.js';
+
+export class OpenTelemetryMetricsFactory implements MetricsFactory {
+  private meter: Meter;
+
+  constructor(meter: Meter) {
+    this.meter = meter;
+  }
+
+  createCounter(metadata: MetricMetadata): Counter {
+    return this.meter.createCounter(metadata.name, {
+      description: metadata.description,
+      unit: metadata.unit,
+      valueType: this.toValueType(metadata.precision)
+    });
+  }
+
+  createObservableGauge(metadata: MetricMetadata): ObservableGauge {
+    const gauge = this.meter.createObservableGauge(metadata.name, {
+      description: metadata.description,
+      unit: metadata.unit,
+      valueType: this.toValueType(metadata.precision)
+    });
+
+    return {
+      setValueProvider(valueProvider: () => Promise<number | undefined>) {
+        gauge.addCallback(async (result) => {
+          const value = await valueProvider();
+
+          if (value) {
+            result.observe(value);
+          }
+        });
+      }
+    };
+  }
+
+  createUpDownCounter(metadata: MetricMetadata): UpDownCounter {
+    return this.meter.createUpDownCounter(metadata.name, {
+      description: metadata.description,
+      unit: metadata.unit,
+      valueType: this.toValueType(metadata.precision)
+    });
+  }
+
+  private toValueType(precision?: Precision): ValueType {
+    if (!precision) {
+      return ValueType.INT;
+    }
+
+    switch (precision) {
+      case Precision.INT:
+        return ValueType.INT;
+      case Precision.DOUBLE:
+        return ValueType.DOUBLE;
+    }
+  }
+}

package/src/metrics/open-telemetry/util.ts

@@ -0,0 +1,80 @@
+import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
+import { Resource } from '@opentelemetry/resources';
+import { ServiceContext } from '../../system/ServiceContext.js';
+import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js';
+import { MetricsFactory } from '../metrics-interfaces.js';
+import { logger } from '@powersync/lib-services-framework';
+
+export interface RuntimeMetadata {
+  [key: string]: string | number | undefined;
+}
+
+export function createOpenTelemetryMetricsFactory(context: ServiceContext): MetricsFactory {
+  const { configuration, lifeCycleEngine, storageEngine } = context;
+  const configuredExporters: MetricReader[] = [];
+
+  if (configuration.telemetry.prometheus_port) {
+    const prometheusExporter = new PrometheusExporter({
+      port: configuration.telemetry.prometheus_port,
+      preventServerStart: true
+    });
+    configuredExporters.push(prometheusExporter);
+
+    lifeCycleEngine.withLifecycle(prometheusExporter, {
+      start: async () => {
+        await prometheusExporter.startServer();
+        logger.info(`Prometheus metric export enabled on port:${configuration.telemetry.prometheus_port}`);
+      }
+    });
+  }
+
+  if (!configuration.telemetry.disable_telemetry_sharing) {
+    const periodicExporter = new PeriodicExportingMetricReader({
+      exporter: new OTLPMetricExporter({
+        url: configuration.telemetry.internal_service_endpoint
+      }),
+      exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
+    });
+
+    configuredExporters.push(periodicExporter);
+  }
+
+  let resolvedMetadata: (metadata: RuntimeMetadata) => void;
+  const runtimeMetadata: Promise<RuntimeMetadata> = new Promise((resolve) => {
+    resolvedMetadata = resolve;
+  });
+
+  lifeCycleEngine.withLifecycle(null, {
+    start: async () => {
+      const bucketStorage = storageEngine.activeBucketStorage;
+      try {
+        const instanceId = await bucketStorage.getPowerSyncInstanceId();
+        resolvedMetadata({ ['instance_id']: instanceId });
+      } catch (err) {
+        resolvedMetadata({ ['instance_id']: 'Unknown' });
+      }
+    }
+  });
+
+  const meterProvider = new MeterProvider({
+    resource: new Resource(
+      {
+        ['service']: 'PowerSync'
+      },
+      runtimeMetadata
+    ),
+    readers: configuredExporters
+  });
+
+  lifeCycleEngine.withLifecycle(meterProvider, {
+    stop: async () => {
+      await meterProvider.shutdown();
+    }
+  });
+
+  const meter = meterProvider.getMeter('powersync');
+
+  return new OpenTelemetryMetricsFactory(meter);
+}

package/src/metrics/register-metrics.ts

@@ -0,0 +1,56 @@
+import { ServiceContextContainer } from '../system/ServiceContext.js';
+import { createOpenTelemetryMetricsFactory } from './open-telemetry/util.js';
+import { MetricsEngine } from './MetricsEngine.js';
+import { createCoreAPIMetrics, initializeCoreAPIMetrics } from '../api/api-metrics.js';
+import { createCoreReplicationMetrics, initializeCoreReplicationMetrics } from '../replication/replication-metrics.js';
+import { createCoreStorageMetrics, initializeCoreStorageMetrics } from '../storage/storage-metrics.js';
+
+export enum MetricModes {
+  API = 'api',
+  REPLICATION = 'replication',
+  STORAGE = 'storage'
+}
+
+export type MetricsRegistrationOptions = {
+  service_context: ServiceContextContainer;
+  modes: MetricModes[];
+};
+
+export const registerMetrics = async (options: MetricsRegistrationOptions) => {
+  const { service_context, modes } = options;
+
+  const metricsFactory = createOpenTelemetryMetricsFactory(service_context);
+  const metricsEngine = new MetricsEngine({
+    factory: metricsFactory,
+    disable_telemetry_sharing: service_context.configuration.telemetry.disable_telemetry_sharing
+  });
+  service_context.register(MetricsEngine, metricsEngine);
+
+  if (modes.includes(MetricModes.API)) {
+    createCoreAPIMetrics(metricsEngine);
+    initializeCoreAPIMetrics(metricsEngine);
+  }
+
+  if (modes.includes(MetricModes.REPLICATION)) {
+    createCoreReplicationMetrics(metricsEngine);
+    initializeCoreReplicationMetrics(metricsEngine);
+  }
+
+  if (modes.includes(MetricModes.STORAGE)) {
+    createCoreStorageMetrics(metricsEngine);
+
+    // This requires an instantiated bucket storage, which is only created when the lifecycle starts
+    service_context.storageEngine.registerListener({
+      storageActivated: (bucketStorage) => {
+        initializeCoreStorageMetrics(metricsEngine, bucketStorage);
+      }
+    });
+  }
+
+  service_context.lifeCycleEngine.withLifecycle(metricsEngine, {
+    start: async () => {
+      await metricsEngine.start();
+    },
+    stop: () => metricsEngine.shutdown()
+  });
+};

package/src/replication/AbstractReplicationJob.ts

@@ -2,10 +2,12 @@ import { container, logger } from '@powersync/lib-services-framework';
 import winston from 'winston';
 import * as storage from '../storage/storage-index.js';
 import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
 
 export interface AbstractReplicationJobOptions {
   id: string;
   storage: storage.SyncRulesBucketStorage;
+  metrics: MetricsEngine;
   lock: storage.ReplicationLock;
   rateLimiter: ErrorRateLimiter;
 }

package/src/replication/AbstractReplicator.ts

@@ -7,6 +7,7 @@ import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider
 import { AbstractReplicationJob } from './AbstractReplicationJob.js';
 import { ErrorRateLimiter } from './ErrorRateLimiter.js';
 import { ConnectionTestResult } from './ReplicationModule.js';
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
 
 // 5 minutes
 const PING_INTERVAL = 1_000_000_000n * 300n;
@@ -19,6 +20,7 @@ export interface CreateJobOptions {
 export interface AbstractReplicatorOptions {
   id: string;
   storageEngine: StorageEngine;
+  metricsEngine: MetricsEngine;
   syncRuleProvider: SyncRulesProvider;
   /**
    * This limits the effect of retries when there is a persistent issue.
@@ -33,6 +35,7 @@ export interface AbstractReplicatorOptions {
  */
 export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
   protected logger: winston.Logger;
+
   /**
    *  Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
    *  transitioning to a new set of sync rules.
@@ -72,6 +75,10 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     return this.options.rateLimiter;
   }
 
+  protected get metrics() {
+    return this.options.metricsEngine;
+  }
+
   public async start(): Promise<void> {
     this.runLoop().catch((e) => {
       this.logger.error('Data source fatal replication error', e);

package/src/replication/ReplicationModule.ts

@@ -64,6 +64,14 @@ export abstract class ReplicationModule<TConfig extends DataSourceConfig>
    */
   protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
 
+  /**
+   *  Any additional initialization specific to the module should be added here. Will be called if necessary after the
+   *  main initialization has been completed
+   *  @param context
+   *  @protected
+   */
+  protected abstract onInitialized(context: system.ServiceContext): Promise<void>;
+
   public abstract testConnection(config: TConfig): Promise<ConnectionTestResult>;
 
   /**
@@ -93,6 +101,8 @@ export abstract class ReplicationModule<TConfig extends DataSourceConfig>
 
     context.replicationEngine?.register(this.createReplicator(context));
     context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
+
+    await this.onInitialized(context);
   }
 
   protected decodeConfig(config: TConfig): void {

package/src/replication/replication-index.ts

@@ -3,3 +3,4 @@ export * from './AbstractReplicator.js';
 export * from './ErrorRateLimiter.js';
 export * from './ReplicationEngine.js';
 export * from './ReplicationModule.js';
+export * from './replication-metrics.js';

package/src/replication/replication-metrics.ts

@@ -0,0 +1,45 @@
+import { MetricsEngine } from '../metrics/metrics-index.js';
+import { ReplicationMetric } from '@powersync/service-types';
+
+/**
+ *  Create and register the core replication metrics.
+ *  @param engine
+ */
+export function createCoreReplicationMetrics(engine: MetricsEngine): void {
+  engine.createCounter({
+    name: ReplicationMetric.DATA_REPLICATED_BYTES,
+    description: 'Uncompressed size of replicated data',
+    unit: 'bytes'
+  });
+
+  engine.createCounter({
+    name: ReplicationMetric.ROWS_REPLICATED,
+    description: 'Total number of replicated rows'
+  });
+
+  engine.createCounter({
+    name: ReplicationMetric.TRANSACTIONS_REPLICATED,
+    description: 'Total number of replicated transactions'
+  });
+
+  engine.createCounter({
+    name: ReplicationMetric.CHUNKS_REPLICATED,
+    description: 'Total number of replication chunks'
+  });
+}
+
+/**
+ *  Initialise the core replication metrics. This should be called after the metrics have been created.
+ *  @param engine
+ */
+export function initializeCoreReplicationMetrics(engine: MetricsEngine): void {
+  const data_replicated_bytes = engine.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES);
+  const rows_replicated_total = engine.getCounter(ReplicationMetric.ROWS_REPLICATED);
+  const transactions_replicated_total = engine.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED);
+  const chunks_replicated_total = engine.getCounter(ReplicationMetric.CHUNKS_REPLICATED);
+
+  data_replicated_bytes.add(0);
+  rows_replicated_total.add(0);
+  transactions_replicated_total.add(0);
+  chunks_replicated_total.add(0);
+}

package/src/routes/endpoints/socket-route.ts

@@ -2,18 +2,19 @@ import { ErrorCode, errors, logger, schema } from '@powersync/lib-services-frame
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { serialize } from 'bson';
 
-import { Metrics } from '../../metrics/Metrics.js';
 import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 import { SocketRouteGenerator } from '../router-socket.js';
 import { SyncRoutes } from './sync-stream.js';
 
+import { APIMetric } from '@powersync/service-types';
+
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
     handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
       const { service_context } = context;
-      const { routerEngine, syncContext } = service_context;
+      const { routerEngine, metricsEngine, syncContext } = service_context;
 
       // Create our own controller that we can abort directly
       const controller = new AbortController();
@@ -69,8 +70,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         controller.abort();
       });
 
-      Metrics.getInstance().concurrent_connections.add(1);
-      const tracker = new sync.RequestTracker();
+      metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
+      const tracker = new sync.RequestTracker(metricsEngine);
       try {
         for await (const data of sync.streamResponse({
           syncContext: syncContext,
@@ -147,7 +148,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
           operations_synced: tracker.operationsSynced,
           data_synced_bytes: tracker.dataSyncedBytes
         });
-        Metrics.getInstance().concurrent_connections.add(-1);
+        metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
       }
     }
   });

package/src/routes/endpoints/sync-stream.ts

@@ -5,10 +5,11 @@ import { Readable } from 'stream';
 import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 
-import { Metrics } from '../../metrics/Metrics.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
+import { APIMetric } from '@powersync/service-types';
+
 export enum SyncRoutes {
   STREAM = '/sync/stream'
 }
@@ -20,7 +21,7 @@ export const syncStreamed = routeDefinition({
   validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
   handler: async (payload) => {
     const { service_context } = payload.context;
-    const { routerEngine, storageEngine, syncContext } = service_context;
+    const { routerEngine, storageEngine, metricsEngine, syncContext } = service_context;
     const headers = payload.request.headers;
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
     const clientId = payload.params.client_id;
@@ -49,9 +50,9 @@ export const syncStreamed = routeDefinition({
     const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions());
 
     const controller = new AbortController();
-    const tracker = new sync.RequestTracker();
+    const tracker = new sync.RequestTracker(metricsEngine);
     try {
-      Metrics.getInstance().concurrent_connections.add(1);
+      metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
       const stream = Readable.from(
         sync.transformToBytesTracked(
           sync.ndjson(
@@ -96,7 +97,7 @@ export const syncStreamed = routeDefinition({
         data: stream,
         afterSend: async () => {
           controller.abort();
-          Metrics.getInstance().concurrent_connections.add(-1);
+          metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
           logger.info(`Sync stream complete`, {
             user_id: syncParams.user_id,
             client_id: clientId,
@@ -108,7 +109,7 @@ export const syncStreamed = routeDefinition({
       });
     } catch (ex) {
       controller.abort();
-      Metrics.getInstance().concurrent_connections.add(-1);
+      metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
     }
   }
 });

package/src/storage/storage-index.ts

@@ -6,6 +6,7 @@ export * from './SourceEntity.js';
 export * from './SourceTable.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';
+export * from './storage-metrics.js';
 export * from './WriteCheckpointAPI.js';
 export * from './BucketStorageFactory.js';
 export * from './BucketStorageBatch.js';