@powersync/service-core 1.15.7 → 1.16.0

package/src/events/EventsEngine.ts

@@ -0,0 +1,38 @@
+import EventEmitter from 'node:events';
+import { logger } from '@powersync/lib-services-framework';
+import { event_types } from '@powersync/service-types';
+
+export class EventsEngine {
+  private emitter: EventEmitter;
+  private events: Set<event_types.EventsEngineEventType> = new Set();
+  constructor() {
+    this.emitter = new EventEmitter({ captureRejections: true });
+    this.emitter.on('error', (error: Error) => {
+      logger.error(error.message);
+    });
+  }
+
+  /**
+   * All new events added need to be subscribed to be used.
+   * @example engine.subscribe(new MyNewEvent(storageEngine));
+   */
+  subscribe<K extends event_types.EventsEngineEventType>(event: event_types.EmitterEvent<K>): void {
+    if (!this.events.has(event.event)) {
+      this.events.add(event.event);
+    }
+    this.emitter.on(event.event, event.handler.bind(event));
+  }
+
+  get listEvents(): event_types.EventsEngineEventType[] {
+    return Array.from(this.events.values());
+  }
+
+  emit<K extends keyof event_types.SubscribeEvents>(event: K, data: event_types.SubscribeEvents[K]): void {
+    this.emitter.emit(event, data);
+  }
+
+  shutDown(): void {
+    logger.info(`Shutting down EmitterEngine and removing all listeners for ${this.listEvents.join(', ')}.`);
+    this.emitter.removeAllListeners();
+  }
+}

package/src/routes/configure-fastify.ts

@@ -1,5 +1,4 @@
 import type fastify from 'fastify';
-import * as uuid from 'uuid';
 
 import { registerFastifyNotFoundHandler, registerFastifyRoutes } from './route-register.js';
 

package/src/routes/endpoints/socket-route.ts

@@ -1,12 +1,11 @@
 import { ErrorCode, errors, schema } from '@powersync/lib-services-framework';
-import { RequestParameters } from '@powersync/service-sync-rules';
 
 import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 import { SocketRouteGenerator } from '../router-socket.js';
 import { SyncRoutes } from './sync-stream.js';
 
-import { APIMetric } from '@powersync/service-types';
+import { APIMetric, event_types } from '@powersync/service-types';
 
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
@@ -14,6 +13,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
     handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal, connection }) => {
       const { service_context, logger } = context;
       const { routerEngine, metricsEngine, syncContext } = service_context;
+      const streamStart = Date.now();
 
       logger.defaultMeta = {
         ...logger.defaultMeta,
@@ -21,7 +21,15 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         client_id: params.client_id,
         user_agent: context.user_agent
       };
-      const streamStart = Date.now();
+
+      const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
+        client_id: params.client_id ?? '',
+        user_id: context.user_id!,
+        user_agent: context.user_agent,
+        // At this point the token_payload is guaranteed to be present
+        jwt_exp: new Date(context.token_payload!.exp * 1000),
+        connected_at: new Date(streamStart)
+      };
 
       // Best effort guess on why the stream was closed.
       // We use the `??=` operator everywhere, so that we catch the first relevant
@@ -83,6 +91,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       });
 
       metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
+      service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData);
       const tracker = new sync.RequestTracker(metricsEngine);
       if (connection.tracker.encoding) {
         // Must be set before we start the stream
@@ -174,6 +183,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
           close_reason: closeReason ?? 'unknown'
         });
         metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
+        service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
+          ...sdkData,
+          disconnected_at: new Date()
+        });
       }
     }
   });

package/src/routes/endpoints/sync-stream.ts

@@ -7,8 +7,8 @@ import * as util from '../../util/util-index.js';
 
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
+import { APIMetric, event_types } from '@powersync/service-types';
 
-import { APIMetric } from '@powersync/service-types';
 import { maybeCompressResponseStream } from '../compression.js';
 
 export enum SyncRoutes {
@@ -25,7 +25,7 @@ export const syncStreamed = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const { service_context, logger } = payload.context;
+    const { service_context, logger, token_payload } = payload.context;
     const { routerEngine, storageEngine, metricsEngine, syncContext } = service_context;
     const headers = payload.request.headers;
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
@@ -44,6 +44,14 @@ export const syncStreamed = routeDefinition({
       user_id: payload.context.user_id,
       bson: useBson
     };
+    const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
+      client_id: clientId ?? '',
+      user_id: payload.context.user_id!,
+      user_agent: userAgent as string,
+      // At this point the token_payload is guaranteed to be present
+      jwt_exp: new Date(token_payload!.exp * 1000),
+      connected_at: new Date(streamStart)
+    };
 
     if (routerEngine.closed) {
       throw new errors.ServiceError({
@@ -69,6 +77,7 @@ export const syncStreamed = routeDefinition({
     const tracker = new sync.RequestTracker(metricsEngine);
     try {
       metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
+      service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData);
       const syncLines = sync.streamResponse({
         syncContext: syncContext,
         bucketStorage,
@@ -134,6 +143,10 @@ export const syncStreamed = routeDefinition({
           }
           controller.abort();
           metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
+          service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
+            ...sdkData,
+            disconnected_at: new Date()
+          });
           logger.info(`Sync stream complete`, {
             ...tracker.getLogMeta(),
             stream_ms: Date.now() - streamStart,
@@ -144,6 +157,10 @@ export const syncStreamed = routeDefinition({
     } catch (ex) {
       controller.abort();
       metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
+      service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
+        ...sdkData,
+        disconnected_at: new Date()
+      });
     }
   }
 });

package/src/routes/router.ts

@@ -1,4 +1,4 @@
-import { router, ServiceError, Logger } from '@powersync/lib-services-framework';
+import { Logger, router } from '@powersync/lib-services-framework';
 import type { JwtPayload } from '../auth/auth-index.js';
 import { ServiceContext } from '../system/ServiceContext.js';
 import { RouterEngine } from './RouterEngine.js';
@@ -31,11 +31,11 @@ export type BasicRouterRequest = {
   hostname: string;
 };
 
-export type ConextProviderOptions = {
+export type ContextProviderOptions = {
   logger: Logger;
 };
 
-export type ContextProvider = (request: BasicRouterRequest, options: ConextProviderOptions) => Promise<Context>;
+export type ContextProvider = (request: BasicRouterRequest, options: ContextProviderOptions) => Promise<Context>;
 
 export type RequestEndpoint<
   I,

package/src/storage/BucketStorageFactory.ts

@@ -3,6 +3,7 @@ import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent }
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { ReplicationLock } from './ReplicationLock.js';
 import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
+import { ReportStorage } from './ReportStorage.js';
 
 /**
  * Represents a configured storage provider.
@@ -164,3 +165,4 @@ export interface TestStorageOptions {
   doNotClear?: boolean;
 }
 export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
+export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;

package/src/storage/ReportStorage.ts

@@ -0,0 +1,39 @@
+import { event_types } from '@powersync/service-types';
+
+/**
+ * Represents a configured report storage.
+ *
+ * Report storage is used for storing localized data for the instance.
+ * Data can then be used for reporting purposes.
+ *
+ */
+export interface ReportStorage extends AsyncDisposable {
+  /**
+   * Report a client connection.
+   */
+  reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void>;
+  /**
+   * Report a client disconnection.
+   */
+  reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void>;
+  /**
+   * Get currently connected clients.
+   * This will return any short or long term connected clients.
+   * Clients that have no disconnected_at timestamp and that have a valid jwt_exp timestamp are considered connected.
+   */
+  getConnectedClients(): Promise<event_types.ClientConnectionReportResponse>;
+  /**
+   * Get a report of client connections over a day, week or month.
+   * This is internally used to generate reports over it always returns the previous day, week or month.
+   * Usually this is call on the start of the new day, week or month. It will return all unique completed connections
+   * as well as uniques currently connected clients.
+   */
+  getClientConnectionReports(
+    data: event_types.ClientConnectionReportRequest
+  ): Promise<event_types.ClientConnectionReportResponse>;
+  /**
+   * Delete old connection data based on a specific date.
+   * This is used to clean up old connection data that is no longer needed.
+   */
+  deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void>;
+}

package/src/storage/StorageEngine.ts

@@ -1,7 +1,7 @@
 import { BaseObserver, logger, ServiceError } from '@powersync/lib-services-framework';
 import { ResolvedPowerSyncConfig } from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
-import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
+import { ActiveStorage, StorageProvider } from './StorageProvider.js';
 
 export type StorageEngineOptions = {
   configuration: ResolvedPowerSyncConfig;
@@ -14,7 +14,7 @@ export interface StorageEngineListener {
 
 export class StorageEngine extends BaseObserver<StorageEngineListener> {
   // TODO: This will need to revisited when we actually support multiple storage providers.
-  private storageProviders: Map<string, BucketStorageProvider> = new Map();
+  private storageProviders: Map<string, StorageProvider> = new Map();
   private currentActiveStorage: ActiveStorage | null = null;
 
   constructor(private options: StorageEngineOptions) {
@@ -37,7 +37,7 @@ export class StorageEngine extends BaseObserver<StorageEngineListener> {
    * Register a provider which generates a {@link BucketStorageFactory}
    * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
    */
-  registerProvider(provider: BucketStorageProvider) {
+  registerProvider(provider: StorageProvider) {
     this.storageProviders.set(provider.type, provider);
   }
 

package/src/storage/StorageProvider.ts

@@ -1,9 +1,11 @@
 import { ServiceError } from '@powersync/lib-services-framework';
 import * as util from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
+import { ReportStorage } from './ReportStorage.js';
 
 export interface ActiveStorage {
   storage: BucketStorageFactory;
+  reportStorage: ReportStorage;
   shutDown(): Promise<void>;
 
   /**
@@ -22,7 +24,7 @@ export interface GetStorageOptions {
 /**
  * Represents a provider that can create a storage instance for a specific storage type from configuration.
  */
-export interface BucketStorageProvider {
+export interface StorageProvider {
   /**
    *  The storage type that this provider provides.
    *  The type should match the `type` field in the config.

package/src/storage/SyncRulesBucketStorage.ts

@@ -65,7 +65,7 @@ export interface SyncRulesBucketStorage
   /**
    * Lightweight "compact" process to populate the checksum cache, if any.
    */
-  populatePersistentChecksumCache(options?: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void>;
+  populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults>;
 
   // ## Read operations
 
@@ -225,6 +225,19 @@ export interface CompactOptions {
   signal?: AbortSignal;
 }
 
+export interface PopulateChecksumCacheOptions {
+  maxOpId: util.InternalOpId;
+  minBucketChanges?: number;
+  signal?: AbortSignal;
+}
+
+export interface PopulateChecksumCacheResults {
+  /**
+   * Number of buckets we have calculated checksums for.
+   */
+  buckets: number;
+}
+
 export interface ClearStorageOptions {
   signal?: AbortSignal;
 }

package/src/storage/storage-index.ts

@@ -13,3 +13,4 @@ export * from './BucketStorageBatch.js';
 export * from './SyncRulesBucketStorage.js';
 export * from './PersistedSyncRulesContent.js';
 export * from './ReplicationLock.js';
+export * from './ReportStorage.js';

package/src/system/ServiceContext.ts

@@ -1,4 +1,4 @@
-import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
+import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework';
 
 import { framework } from '../index.js';
 import * as metrics from '../metrics/MetricsEngine.js';
@@ -8,6 +8,7 @@ import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
 import { SyncContext } from '../sync/SyncContext.js';
 import * as utils from '../util/util-index.js';
+import { EventsEngine } from '../events/EventsEngine.js';
 
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
@@ -19,6 +20,7 @@ export interface ServiceContext {
   migrations: PowerSyncMigrationManager;
   syncContext: SyncContext;
   serviceMode: ServiceContextMode;
+  eventsEngine: EventsEngine;
 }
 
 export enum ServiceContextMode {
@@ -45,6 +47,7 @@ export class ServiceContextContainer implements ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
   lifeCycleEngine: LifeCycledSystem;
   storageEngine: storage.StorageEngine;
+  eventsEngine: EventsEngine;
   syncContext: SyncContext;
   routerEngine: routes.RouterEngine;
   serviceMode: ServiceContextMode;
@@ -66,6 +69,11 @@ export class ServiceContextContainer implements ServiceContext {
       }
     });
 
+    this.eventsEngine = new EventsEngine();
+    this.lifeCycleEngine.withLifecycle(this.eventsEngine, {
+      stop: (emitterEngine) => emitterEngine.shutDown()
+    });
+
     this.lifeCycleEngine.withLifecycle(this.storageEngine, {
       start: (storageEngine) => storageEngine.start(),
       stop: (storageEngine) => storageEngine.shutDown()
@@ -89,6 +97,10 @@ export class ServiceContextContainer implements ServiceContext {
       // Migrations should be executed before the system starts
       start: () => migrationManager[Symbol.asyncDispose]()
     });
+
+    this.lifeCycleEngine.withLifecycle(this.eventsEngine, {
+      stop: (emitterEngine) => emitterEngine.shutDown()
+    });
   }
 
   get replicationEngine(): replication.ReplicationEngine | null {

package/test/src/routes/mocks.ts

@@ -11,6 +11,7 @@ import {
   SyncRulesBucketStorage
 } from '@/index.js';
 import { MeterProvider } from '@opentelemetry/sdk-metrics';
+import { EventsEngine } from '@/events/EventsEngine.js';
 
 export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
   // This is very incomplete - just enough to get the current tests passing.
@@ -34,6 +35,7 @@ export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | nu
   createCoreAPIMetrics(metricsEngine);
   const service_context: Partial<ServiceContext> = {
     syncContext: new SyncContext({ maxBuckets: 1, maxDataFetchConcurrency: 1, maxParameterQueryResults: 1 }),
+    eventsEngine: new EventsEngine(),
     routerEngine: {
       getAPI() {
         return {

package/test/src/routes/stream.test.ts

@@ -3,7 +3,7 @@ import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-fr
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { Readable, Writable } from 'stream';
 import { pipeline } from 'stream/promises';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { describe, expect, it } from 'vitest';
 import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
 import { mockServiceContext } from './mocks.js';
 
@@ -12,7 +12,12 @@ describe('Stream Route', () => {
     it('handles missing sync rules', async () => {
       const context: Context = {
         logger: logger,
-        service_context: mockServiceContext(null)
+        service_context: mockServiceContext(null),
+        token_payload: {
+          sub: '',
+          exp: 0,
+          iat: 0
+        }
       };
 
       const request: BasicRouterRequest = {
@@ -21,10 +26,13 @@ describe('Stream Route', () => {
         protocol: 'http'
       };
 
-      const error = (await (syncStreamed.handler({ context, params: {}, request }) as Promise<RouterResponse>).catch(
-        (e) => e
-      )) as ServiceError;
-
+      const error = (await (
+        syncStreamed.handler({
+          context,
+          params: {},
+          request
+        }) as Promise<RouterResponse>
+      ).catch((e) => e)) as ServiceError;
       expect(error.errorData.status).toEqual(500);
       expect(error.errorData.code).toEqual('PSYNC_S2302');
     });