@powersync/service-core 0.0.0-dev-20240708120322 → 0.0.0-dev-20240718134716

package/src/metrics/Metrics.ts

@@ -1,12 +1,9 @@
 import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
 import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
-import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
 import * as jpgwire from '@powersync/service-jpgwire';
-import * as util from '../util/util-index.js';
 import * as storage from '../storage/storage-index.js';
 import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
-import { Resource } from '@opentelemetry/resources';
 import { logger } from '@powersync/lib-services-framework';
 
 export interface MetricsOptions {
@@ -16,8 +13,6 @@ export interface MetricsOptions {
 }
 
 export class Metrics {
-  private static instance: Metrics;
-
   private prometheusExporter: PrometheusExporter;
   private meterProvider: MeterProvider;
 
@@ -60,7 +55,7 @@ export class Metrics {
   // Record on API pod
   public concurrent_connections: UpDownCounter<Attributes>;
 
-  private constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
+  constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
     this.meterProvider = meterProvider;
     this.prometheusExporter = prometheusExporter;
     const meter = meterProvider.getMeter('powersync');
@@ -132,66 +127,6 @@ export class Metrics {
     this.concurrent_connections.add(0);
   }
 
-  public static getInstance(): Metrics {
-    if (!Metrics.instance) {
-      throw new Error('Metrics have not been initialised');
-    }
-
-    return Metrics.instance;
-  }
-
-  public static async initialise(options: MetricsOptions): Promise<void> {
-    if (Metrics.instance) {
-      return;
-    }
-    logger.info('Configuring telemetry.');
-
-    logger.info(
-      `
-Attention:
-PowerSync collects completely anonymous telemetry regarding usage.
-This information is used to shape our roadmap to better serve our customers.
-You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
-https://docs.powersync.com/self-hosting/telemetry
-Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'}
-    `.trim()
-    );
-
-    const configuredExporters: MetricReader[] = [];
-
-    const port: number = util.env.METRICS_PORT ?? 0;
-    const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true });
-    configuredExporters.push(prometheusExporter);
-
-    if (!options.disable_telemetry_sharing) {
-      logger.info('Sharing anonymous telemetry');
-      const periodicExporter = new PeriodicExportingMetricReader({
-        exporter: new OTLPMetricExporter({
-          url: options.internal_metrics_endpoint
-        }),
-        exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
-      });
-
-      configuredExporters.push(periodicExporter);
-    }
-
-    const meterProvider = new MeterProvider({
-      resource: new Resource({
-        ['service']: 'PowerSync',
-        ['instance_id']: options.powersync_instance_id
-      }),
-      readers: configuredExporters
-    });
-
-    if (port > 0) {
-      await prometheusExporter.startServer();
-    }
-
-    Metrics.instance = new Metrics(meterProvider, prometheusExporter);
-
-    logger.info('Telemetry configuration complete.');
-  }
-
   public async shutdown(): Promise<void> {
     await this.meterProvider.shutdown();
   }

package/src/migrations/migrations.ts

@@ -8,6 +8,7 @@ import * as locks from '../locks/locks-index.js';
 import { Direction } from './definitions.js';
 import { createMongoMigrationStore } from './store/migration-store.js';
 import { execute, writeLogsToStore } from './executor.js';
+import { logger } from '@powersync/lib-services-framework';
 
 const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
 const MONGO_LOCK_PROCESS = 'migrations';
@@ -63,6 +64,7 @@ export const migrate = async (options: MigrationOptions) => {
   const { storage } = config;
 
   const client = db.mongo.createMongoClient(storage);
+  logger.info('Connecting to MongoDB');
   await client.connect();
 
   const clientDB = client.db(storage.database);
@@ -73,6 +75,7 @@ export const migrate = async (options: MigrationOptions) => {
   });
 
   // Only one process should execute this at a time.
+  logger.info('Acquiring lock');
   const lockId = await manager.acquire();
 
   if (!lockId) {
@@ -92,6 +95,7 @@ export const migrate = async (options: MigrationOptions) => {
   process.addListener('beforeExit', releaseLock);
 
   try {
+    logger.info('Loading migrations');
     const migrations = await loadMigrations(MIGRATIONS_DIR, runner_config);
 
     // Use the provided config to connect to Mongo
@@ -99,6 +103,7 @@ export const migrate = async (options: MigrationOptions) => {
 
     const state = await store.load();
 
+    logger.info('Running migrations');
     const logStream = execute({
       direction: direction,
       migrations,
@@ -111,8 +116,11 @@ export const migrate = async (options: MigrationOptions) => {
       state
     });
   } finally {
+    logger.info('Releasing lock');
     await releaseLock();
+    logger.info('Closing database');
     await client.close(true);
     process.removeListener('beforeExit', releaseLock);
+    logger.info('Done with migrations');
   }
 };

package/src/replication/WalStream.ts

@@ -406,7 +406,7 @@ WHERE  oid = $1::regclass`,
         await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record });
       }
       at += rows.length;
-      Metrics.getInstance().rows_replicated_total.add(rows.length);
+      container.getImplementation(Metrics).rows_replicated_total.add(rows.length);
 
       await touch();
     }
@@ -492,19 +492,21 @@ WHERE  oid = $1::regclass`,
         return null;
       }
 
+      const metrics = container.getImplementation(Metrics);
+
       if (msg.tag == 'insert') {
-        Metrics.getInstance().rows_replicated_total.add(1);
+        metrics.rows_replicated_total.add(1);
         const baseRecord = util.constructAfterRecord(msg);
         return await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: baseRecord });
       } else if (msg.tag == 'update') {
-        Metrics.getInstance().rows_replicated_total.add(1);
+        metrics.rows_replicated_total.add(1);
         // "before" may be null if the replica id columns are unchanged
         // It's fine to treat that the same as an insert.
         const before = util.constructBeforeRecord(msg);
         const after = util.constructAfterRecord(msg);
         return await batch.save({ tag: 'update', sourceTable: table, before: before, after: after });
       } else if (msg.tag == 'delete') {
-        Metrics.getInstance().rows_replicated_total.add(1);
+        metrics.rows_replicated_total.add(1);
         const before = util.constructBeforeRecord(msg)!;
 
         return await batch.save({ tag: 'delete', sourceTable: table, before: before, after: undefined });
@@ -555,6 +557,8 @@ WHERE  oid = $1::regclass`,
     // Auto-activate as soon as initial replication is done
     await this.storage.autoActivate();
 
+    const metrics = container.getImplementation(Metrics);
+
     await this.storage.startBatch({}, async (batch) => {
       // Replication never starts in the middle of a transaction
       let inTx = false;
@@ -577,7 +581,7 @@ WHERE  oid = $1::regclass`,
           } else if (msg.tag == 'begin') {
             inTx = true;
           } else if (msg.tag == 'commit') {
-            Metrics.getInstance().transactions_replicated_total.add(1);
+            metrics.transactions_replicated_total.add(1);
             inTx = false;
             await batch.commit(msg.lsn!);
             await this.ack(msg.lsn!, replicationStream);
@@ -602,7 +606,7 @@ WHERE  oid = $1::regclass`,
           }
         }
 
-        Metrics.getInstance().chunks_replicated_total.add(1);
+        metrics.chunks_replicated_total.add(1);
       }
     });
   }

package/src/routes/endpoints/socket-route.ts

@@ -1,4 +1,4 @@
-import { errors, logger, schema } from '@powersync/lib-services-framework';
+import { container, errors, logger, schema } from '@powersync/lib-services-framework';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { serialize } from 'bson';
 
@@ -7,6 +7,7 @@ import { streamResponse } from '../../sync/sync.js';
 import * as util from '../../util/util-index.js';
 import { SocketRouteGenerator } from '../router-socket.js';
 import { SyncRoutes } from './sync-stream.js';
+import { RequestTracker } from '../../sync/RequestTracker.js';
 
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
@@ -65,7 +66,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         observer.triggerCancel();
       });
 
-      Metrics.getInstance().concurrent_connections.add(1);
+      const metrics = container.getImplementation(Metrics);
+
+      metrics.concurrent_connections.add(1);
+      const tracker = new RequestTracker();
       try {
         for await (const data of streamResponse({
           storage,
@@ -79,6 +83,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             // RSocket handles keepalive events by default
             keep_alive: false
           },
+          tracker,
           signal: controller.signal
         })) {
           if (data == null) {
@@ -94,7 +99,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             const serialized = serialize(data) as Buffer;
             responder.onNext({ data: serialized }, false);
             requestedN--;
-            Metrics.getInstance().data_synced_bytes.add(serialized.length);
+            tracker.addDataSynced(serialized.length);
           }
 
           if (requestedN <= 0) {
@@ -126,7 +131,12 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         responder.onComplete();
         removeStopHandler();
         disposer();
-        Metrics.getInstance().concurrent_connections.add(-1);
+        logger.info(`Sync stream complete`, {
+          user_id: syncParams.user_id,
+          operations_synced: tracker.operationsSynced,
+          data_synced_bytes: tracker.dataSyncedBytes
+        });
+        metrics.concurrent_connections.add(-1);
       }
     }
   });

package/src/routes/endpoints/sync-stream.ts

@@ -1,4 +1,4 @@
-import { errors, logger, router, schema } from '@powersync/lib-services-framework';
+import { container, errors, logger, router, schema } from '@powersync/lib-services-framework';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { Readable } from 'stream';
 
@@ -8,6 +8,7 @@ import * as util from '../../util/util-index.js';
 import { Metrics } from '../../metrics/Metrics.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
+import { RequestTracker } from '../../sync/RequestTracker.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -42,9 +43,11 @@ export const syncStreamed = routeDefinition({
         description: 'No sync rules available'
       });
     }
+    const metrics = container.getImplementation(Metrics);
     const controller = new AbortController();
+    const tracker = new RequestTracker();
     try {
-      Metrics.getInstance().concurrent_connections.add(1);
+      metrics.concurrent_connections.add(1);
       const stream = Readable.from(
         sync.transformToBytesTracked(
           sync.ndjson(
@@ -53,9 +56,11 @@ export const syncStreamed = routeDefinition({
               params,
               syncParams,
               token: payload.context.token_payload!,
+              tracker,
               signal: controller.signal
             })
-          )
+          ),
+          tracker
         ),
         { objectMode: false, highWaterMark: 16 * 1024 }
       );
@@ -85,12 +90,17 @@ export const syncStreamed = routeDefinition({
         data: stream,
         afterSend: async () => {
           controller.abort();
-          Metrics.getInstance().concurrent_connections.add(-1);
+          metrics.concurrent_connections.add(-1);
+          logger.info(`Sync stream complete`, {
+            user_id: syncParams.user_id,
+            operations_synced: tracker.operationsSynced,
+            data_synced_bytes: tracker.dataSyncedBytes
+          });
         }
       });
     } catch (ex) {
       controller.abort();
-      Metrics.getInstance().concurrent_connections.add(-1);
+      metrics.concurrent_connections.add(-1);
     }
   }
 });

package/src/routes/route-register.ts

@@ -63,6 +63,7 @@ export function registerFastifyRoutes(
             }
           } catch (ex) {
             const journeyError = errors.JourneyError.isJourneyError(ex) ? ex : new errors.InternalServerError(ex);
+            logger.error(`Request failed`, journeyError);
 
             response = new router.RouterResponse({
               status: journeyError.errorData.status || 500,

package/src/sync/RequestTracker.ts

@@ -0,0 +1,21 @@
+import { container } from '@powersync/lib-services-framework';
+import { Metrics } from '../metrics/Metrics.js';
+
+/**
+ * Record sync stats per request stream.
+ */
+export class RequestTracker {
+  operationsSynced = 0;
+  dataSyncedBytes = 0;
+
+  addOperationsSynced(operations: number) {
+    this.operationsSynced += operations;
+    container.getImplementation(Metrics).operations_synced_total.add(operations);
+  }
+
+  addDataSynced(bytes: number) {
+    this.dataSyncedBytes += bytes;
+
+    container.getImplementation(Metrics).data_synced_bytes.add(bytes);
+  }
+}

package/src/sync/sync.ts

@@ -8,9 +8,9 @@ import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import { logger } from '@powersync/lib-services-framework';
-import { Metrics } from '../metrics/Metrics.js';
 import { mergeAsyncIterables } from './merge.js';
 import { TokenStreamOptions, tokenStream } from './util.js';
+import { RequestTracker } from './RequestTracker.js';
 
 /**
  * Maximum number of connections actively fetching data.
@@ -28,12 +28,14 @@ export interface SyncStreamParameters {
    */
   signal?: AbortSignal;
   tokenStreamOptions?: Partial<TokenStreamOptions>;
+
+  tracker: RequestTracker;
 }
 
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { storage, params, syncParams, token, tokenStreamOptions, signal } = options;
+  const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {
@@ -49,7 +51,7 @@ export async function* streamResponse(
     }
   }
   const ki = tokenStream(token, controller.signal, tokenStreamOptions);
-  const stream = streamResponseInner(storage, params, syncParams, controller.signal);
+  const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
 
@@ -72,6 +74,7 @@ async function* streamResponseInner(
   storage: storage.BucketStorageFactory,
   params: util.StreamingSyncRequest,
   syncParams: RequestParameters,
+  tracker: RequestTracker,
   signal: AbortSignal
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
   // Bucket state of bucket id -> op_id.
@@ -109,6 +112,11 @@ async function* streamResponseInner(
     });
 
     if (allBuckets.length > 1000) {
+      logger.error(`Too many buckets`, {
+        checkpoint,
+        user_id: syncParams.user_id,
+        buckets: allBuckets.length
+      });
       // TODO: Limit number of buckets even before we get to this point
       throw new Error(`Too many buckets: ${allBuckets.length}`);
     }
@@ -137,11 +145,18 @@ async function* streamResponseInner(
       }
       bucketsToFetch = diff.updatedBuckets.map((c) => c.bucket);
 
-      let message = `Updated checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
+      let message = `Updated checkpoint: ${checkpoint} | `;
+      message += `write: ${writeCheckpoint} | `;
       message += `buckets: ${allBuckets.length} | `;
       message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
-      message += `removed: ${limitedBuckets(diff.removedBuckets, 20)} | `;
-      logger.info(message);
+      message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
+      logger.info(message, {
+        checkpoint,
+        user_id: syncParams.user_id,
+        buckets: allBuckets.length,
+        updated: diff.updatedBuckets.length,
+        removed: diff.removedBuckets.length
+      });
 
       const checksum_line: util.StreamingSyncCheckpointDiff = {
         checkpoint_diff: {
@@ -156,7 +171,7 @@ async function* streamResponseInner(
     } else {
       let message = `New checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
       message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
-      logger.info(message);
+      logger.info(message, { checkpoint, user_id: syncParams.user_id, buckets: allBuckets.length });
       bucketsToFetch = allBuckets;
       const checksum_line: util.StreamingSyncCheckpoint = {
         checkpoint: {
@@ -172,7 +187,16 @@ async function* streamResponseInner(
 
     // This incrementally updates dataBuckets with each individual bucket position.
     // At the end of this, we can be sure that all buckets have data up to the checkpoint.
-    yield* bucketDataInBatches({ storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal });
+    yield* bucketDataInBatches({
+      storage,
+      checkpoint,
+      bucketsToFetch,
+      dataBuckets,
+      raw_data,
+      binary_data,
+      signal,
+      tracker
+    });
 
     await new Promise((resolve) => setTimeout(resolve, 10));
   }
@@ -186,6 +210,7 @@ interface BucketDataRequest {
   dataBuckets: Map<string, string>;
   raw_data: boolean | undefined;
   binary_data: boolean | undefined;
+  tracker: RequestTracker;
   signal: AbortSignal;
 }
 
@@ -221,11 +246,16 @@ async function* bucketDataInBatches(request: BucketDataRequest) {
   }
 }
 
+interface BucketDataBatchResult {
+  done: boolean;
+  data: any;
+}
+
 /**
  * Extracted as a separate internal function just to avoid memory leaks.
  */
-async function* bucketDataBatch(request: BucketDataRequest) {
-  const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal } = request;
+async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<BucketDataBatchResult, void> {
+  const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, tracker, signal } = request;
 
   const [_, release] = await syncSemaphore.acquire();
   try {
@@ -272,7 +302,7 @@ async function* bucketDataBatch(request: BucketDataRequest) {
         // iterator memory in case if large data sent.
         yield { data: null, done: false };
       }
-      Metrics.getInstance().operations_synced_total.add(r.data.length);
+      tracker.addOperationsSynced(r.data.length);
 
       dataBuckets.set(r.bucket, r.next_after);
     }

package/src/sync/util.ts

@@ -1,7 +1,7 @@
 import * as timers from 'timers/promises';
 
 import * as util from '../util/util-index.js';
-import { Metrics } from '../metrics/Metrics.js';
+import { RequestTracker } from './RequestTracker.js';
 
 export type TokenStreamOptions = {
   /**
@@ -89,10 +89,13 @@ export async function* ndjson(iterator: AsyncIterable<string | null | Record<str
   }
 }
 
-export async function* transformToBytesTracked(iterator: AsyncIterable<string>): AsyncGenerator<Buffer> {
+export async function* transformToBytesTracked(
+  iterator: AsyncIterable<string>,
+  tracker: RequestTracker
+): AsyncGenerator<Buffer> {
   for await (let data of iterator) {
     const encoded = Buffer.from(data, 'utf8');
-    Metrics.getInstance().data_synced_bytes.add(encoded.length);
+    tracker.addDataSynced(encoded.length);
     yield encoded;
   }
 }

package/src/util/config/collectors/config-collector.ts

@@ -1,8 +1,6 @@
-import * as t from 'ts-codec';
 import * as yaml from 'yaml';
 
 import { configFile } from '@powersync/service-types';
-import { schema } from '@powersync/lib-services-framework';
 
 import { RunnerConfig } from '../types.js';
 
@@ -23,13 +21,6 @@ export enum ConfigFileFormat {
  */
 const YAML_ENV_PREFIX = 'PS_';
 
-// ts-codec itself doesn't give great validation errors, so we use json schema for that
-const configSchemaValidator = schema
-  .parseJSONSchema(
-    t.generateJSONSchema(configFile.powerSyncConfig, { allowAdditional: true, parsers: [configFile.portParser] })
-  )
-  .validator();
-
 export abstract class ConfigCollector {
   abstract get name(): string;
 
@@ -39,45 +30,6 @@ export abstract class ConfigCollector {
    */
   abstract collectSerialized(runnerConfig: RunnerConfig): Promise<configFile.SerializedPowerSyncConfig | null>;
 
-  /**
-   * Collects the PowerSyncConfig settings.
-   * Validates and decodes the config.
-   * @returns null if this collector cannot provide a config
-   */
-  async collect(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig | null> {
-    const serialized = await this.collectSerialized(runner_config);
-    if (!serialized) {
-      return null;
-    }
-
-    /**
-     * After this point a serialized config has been found. Any failures to decode or validate
-     * will result in a hard stop.
-     */
-    const decoded = this.decode(serialized);
-    this.validate(decoded);
-    return decoded;
-  }
-
-  /**
-   * Validates input config
-   * ts-codec itself doesn't give great validation errors, so we use json schema for that
-   */
-  validate(config: configFile.PowerSyncConfig) {
-    const valid = configSchemaValidator.validate(config);
-    if (!valid.valid) {
-      throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
-    }
-  }
-
-  decode(encoded: configFile.SerializedPowerSyncConfig): configFile.PowerSyncConfig {
-    try {
-      return configFile.powerSyncConfig.decode(encoded);
-    } catch (ex) {
-      throw new Error(`Failed to decode PowerSync config: ${ex}`);
-    }
-  }
-
   protected parseContent(content: string, contentType?: ConfigFileFormat) {
     switch (contentType) {
       case ConfigFileFormat.YAML: