@powersync/service-core 1.10.1 → 1.11.0

package/src/storage/storage-metrics.ts

@@ -0,0 +1,67 @@
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
+import { logger } from '@powersync/lib-services-framework';
+import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
+import { StorageMetric } from '@powersync/service-types';
+
+export function createCoreStorageMetrics(engine: MetricsEngine): void {
+  engine.createObservableGauge({
+    name: StorageMetric.REPLICATION_SIZE_BYTES,
+    description: 'Size of current data stored in PowerSync',
+    unit: 'bytes'
+  });
+
+  engine.createObservableGauge({
+    name: StorageMetric.OPERATION_SIZE_BYTES,
+    description: 'Size of operations stored in PowerSync',
+    unit: 'bytes'
+  });
+
+  engine.createObservableGauge({
+    name: StorageMetric.PARAMETER_SIZE_BYTES,
+    description: 'Size of parameter data stored in PowerSync',
+    unit: 'bytes'
+  });
+}
+
+export function initializeCoreStorageMetrics(engine: MetricsEngine, storage: BucketStorageFactory): void {
+  const replication_storage_size_bytes = engine.getObservableGauge(StorageMetric.REPLICATION_SIZE_BYTES);
+  const operation_storage_size_bytes = engine.getObservableGauge(StorageMetric.OPERATION_SIZE_BYTES);
+  const parameter_storage_size_bytes = engine.getObservableGauge(StorageMetric.PARAMETER_SIZE_BYTES);
+
+  const MINIMUM_INTERVAL = 60_000;
+
+  let cachedRequest: Promise<StorageMetrics | null> | undefined = undefined;
+  let cacheTimestamp = 0;
+
+  const getMetrics = () => {
+    if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
+      cachedRequest = storage.getStorageMetrics().catch((e) => {
+        logger.error(`Failed to get storage metrics`, e);
+        return null;
+      });
+      cacheTimestamp = Date.now();
+    }
+    return cachedRequest;
+  };
+
+  replication_storage_size_bytes.setValueProvider(async () => {
+    const metrics = await getMetrics();
+    if (metrics) {
+      return metrics.replication_size_bytes;
+    }
+  });
+
+  operation_storage_size_bytes.setValueProvider(async () => {
+    const metrics = await getMetrics();
+    if (metrics) {
+      return metrics.operations_size_bytes;
+    }
+  });
+
+  parameter_storage_size_bytes.setValueProvider(async () => {
+    const metrics = await getMetrics();
+    if (metrics) {
+      return metrics.parameters_size_bytes;
+    }
+  });
+}

package/src/sync/RequestTracker.ts

@@ -1,4 +1,6 @@
-import { Metrics } from '../metrics/Metrics.js';
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
+
+import { APIMetric } from '@powersync/service-types';
 
 /**
  * Record sync stats per request stream.
@@ -7,15 +9,19 @@ export class RequestTracker {
   operationsSynced = 0;
   dataSyncedBytes = 0;
 
+  constructor(private metrics: MetricsEngine) {
+    this.metrics = metrics;
+  }
+
   addOperationsSynced(operations: number) {
     this.operationsSynced += operations;
 
-    Metrics.getInstance().operations_synced_total.add(operations);
+    this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED).add(operations);
   }
 
   addDataSynced(bytes: number) {
     this.dataSyncedBytes += bytes;
 
-    Metrics.getInstance().data_synced_bytes.add(bytes);
+    this.metrics.getCounter(APIMetric.DATA_SYNCED_BYTES).add(bytes);
   }
 }

package/src/sync/sync.ts

@@ -180,7 +180,14 @@ async function* streamResponseInner(
       function markOperationsSent(operations: number) {
         syncedOperations += operations;
         tracker.addOperationsSynced(operations);
-        maybeRaceForNewCheckpoint();
+        // Disable interrupting checkpoints for now.
+        // There is a bug with interrupting checkpoints where:
+        // 1. User is in the middle of syncing a large batch of data (for example initial sync).
+        // 2. A new checkpoint comes in, which interrupts the batch.
+        // 3. However, the new checkpoint does not contain any new data for this connection, so nothing further is synced.
+        // This then causes the client to wait indefinitely for the remaining data or checkpoint_complete message. That is
+        // only resolved when a new checkpoint comes in that does have data for this connection, or the connection is restarted.
+        // maybeRaceForNewCheckpoint();
       }
 
       // This incrementally updates dataBuckets with each individual bucket position.

package/src/system/ServiceContext.ts

@@ -1,7 +1,7 @@
 import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
 
 import { framework } from '../index.js';
-import * as metrics from '../metrics/Metrics.js';
+import * as metrics from '../metrics/MetricsEngine.js';
 import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
 import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
@@ -12,7 +12,7 @@ import { SyncContext } from '../sync/SyncContext.js';
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
   lifeCycleEngine: LifeCycledSystem;
-  metrics: metrics.Metrics | null;
+  metricsEngine: metrics.MetricsEngine;
   replicationEngine: replication.ReplicationEngine | null;
   routerEngine: routes.RouterEngine | null;
   storageEngine: storage.StorageEngine;
@@ -37,6 +37,11 @@ export class ServiceContextContainer implements ServiceContext {
       configuration
     });
 
+    this.lifeCycleEngine.withLifecycle(this.storageEngine, {
+      start: (storageEngine) => storageEngine.start(),
+      stop: (storageEngine) => storageEngine.shutDown()
+    });
+
     this.syncContext = new SyncContext({
       maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency,
       maxBuckets: configuration.api_parameters.max_buckets_per_connection,
@@ -65,8 +70,8 @@ export class ServiceContextContainer implements ServiceContext {
     return container.getOptional(routes.RouterEngine);
   }
 
-  get metrics(): metrics.Metrics | null {
-    return container.getOptional(metrics.Metrics);
+  get metricsEngine(): metrics.MetricsEngine {
+    return container.getImplementation(metrics.MetricsEngine);
   }
 
   get migrations(): PowerSyncMigrationManager {

package/src/util/config/compound-config-collector.ts

@@ -154,6 +154,7 @@ export class CompoundConfigCollector {
       metadata: baseConfig.metadata ?? {},
       migrations: baseConfig.migrations,
       telemetry: {
+        prometheus_port: baseConfig.telemetry?.prometheus_port,
         disable_telemetry_sharing: baseConfig.telemetry?.disable_telemetry_sharing ?? false,
         internal_service_endpoint:
           baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'

package/src/util/config/types.ts

@@ -55,6 +55,7 @@ export type ResolvedPowerSyncConfig = {
   };
 
   telemetry: {
+    prometheus_port?: number;
     disable_telemetry_sharing: boolean;
     internal_service_endpoint: string;
   };

package/src/util/env.ts

@@ -19,10 +19,6 @@ export const env = utils.collectEnvironmentVariables({
    * Runner to be started in this process
    */
   PS_RUNNER_TYPE: utils.type.string.default(ServiceRunner.UNIFIED),
-  /**
-   * Port for metrics
-   */
-  METRICS_PORT: utils.type.number.optional(),
 
   NODE_ENV: utils.type.string.optional()
 });