@powersync/service-core 0.8.7 → 0.9.0

package/src/replication/ReplicationEngine.ts

@@ -0,0 +1,43 @@
+import { logger } from '@powersync/lib-services-framework';
+import { AbstractReplicator } from './AbstractReplicator.js';
+
+export class ReplicationEngine {
+  private readonly replicators: Map<string, AbstractReplicator> = new Map();
+
+  /**
+   *  Register a Replicator with the engine
+   *
+   *  @param replicator
+   */
+  public register(replicator: AbstractReplicator) {
+    if (this.replicators.has(replicator.id)) {
+      throw new Error(`Replicator with id: ${replicator.id} already registered`);
+    }
+    logger.info(`Successfully registered Replicator ${replicator.id} with ReplicationEngine`);
+    this.replicators.set(replicator.id, replicator);
+  }
+
+  /**
+   *  Start replication on all managed Replicators
+   */
+  public start(): void {
+    logger.info('Starting Replication Engine...');
+    for (const replicator of this.replicators.values()) {
+      logger.info(`Starting Replicator: ${replicator.id}`);
+      replicator.start();
+    }
+    logger.info('Successfully started Replication Engine.');
+  }
+
+  /**
+   *  Stop replication on all managed Replicators
+   */
+  public async shutDown(): Promise<void> {
+    logger.info('Shutting down Replication Engine...');
+    for (const replicator of this.replicators.values()) {
+      logger.info(`Stopping Replicator: ${replicator.id}`);
+      await replicator.stop();
+    }
+    logger.info('Successfully shut down Replication Engine.');
+  }
+}

package/src/replication/ReplicationModule.ts

@@ -0,0 +1,122 @@
+import { DataSourceConfig } from '@powersync/service-types/dist/config/PowerSyncConfig.js';
+import * as t from 'ts-codec';
+
+import * as types from '@powersync/service-types';
+import * as api from '../api/api-index.js';
+import * as modules from '../modules/modules-index.js';
+import * as system from '../system/system-index.js';
+import { schema } from '@powersync/lib-services-framework';
+import { AbstractReplicator } from './AbstractReplicator.js';
+import { TearDownOptions } from '../modules/modules-index.js';
+
+/**
+ *  Provides a common interface for testing the connection to a DataSource.
+ */
+export interface ConnectionTester<TConfig extends DataSourceConfig> {
+  /**
+   *  Confirm if a connection can be established to the datasource for the provided datasource configuration
+   *  @param config
+   */
+  testConnection(config: TConfig): Promise<void>;
+}
+
+export interface ReplicationModuleOptions extends modules.AbstractModuleOptions {
+  type: string;
+  configSchema: t.AnyCodec;
+}
+
+/**
+ *  A replication module describes all the functionality that PowerSync requires to
+ *  replicate data from a DataSource. Whenever a new data source is added to powersync this class should be extended.
+ */
+export abstract class ReplicationModule<TConfig extends DataSourceConfig>
+  extends modules.AbstractModule
+  implements ConnectionTester<TConfig>
+{
+  protected type: string;
+  protected configSchema: t.AnyCodec;
+  protected decodedConfig: TConfig | undefined;
+
+  /**
+   * @protected
+   * @param options
+   */
+  protected constructor(options: ReplicationModuleOptions) {
+    super(options);
+    this.type = options.type;
+    this.configSchema = options.configSchema;
+  }
+
+  /**
+   *  Create the RouteAPI adapter for the DataSource required to service the sync API
+   *  endpoints.
+   */
+  protected abstract createRouteAPIAdapter(): api.RouteAPI;
+
+  /**
+   *  Create the Replicator to be used by the ReplicationEngine.
+   */
+  protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
+
+  public abstract testConnection(config: TConfig): Promise<void>;
+
+  /**
+   *  Register this module's Replicators and RouteAPI adapters if the required configuration is present.
+   */
+  public async initialize(context: system.ServiceContext): Promise<void> {
+    if (!context.configuration.connections) {
+      // No data source configuration found in the config skip for now
+      return;
+    }
+
+    const matchingConfig = context.configuration.connections.filter((dataSource) => dataSource.type === this.type);
+    if (!matchingConfig.length) {
+      // No configuration for this module was found
+      return;
+    }
+
+    if (matchingConfig.length > 1) {
+      this.logger.warning(
+        `Multiple data sources of type ${this.type} found in the configuration. Only the first will be used.`
+      );
+    }
+
+    try {
+      const baseMatchingConfig = matchingConfig[0] as TConfig;
+      // If decoding fails, log the error and continue, no replication will happen for this data source
+      this.decodeConfig(baseMatchingConfig);
+
+      context.replicationEngine?.register(this.createReplicator(context));
+      context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
+    } catch (e) {
+      this.logger.error('Failed to initialize.', e);
+    }
+  }
+
+  protected decodeConfig(config: TConfig): void {
+    this.validateConfig(config);
+    this.decodedConfig = this.configSchema.decode(config);
+  }
+
+  private validateConfig(config: TConfig): void {
+    const validator = schema
+      .parseJSONSchema(
+        // This generates a schema for the encoded form of the codec
+        t.generateJSONSchema(this.configSchema, {
+          allowAdditional: true,
+          parsers: [types.configFile.portParser]
+        })
+      )
+      .validator();
+
+    const valid = validator.validate(config);
+
+    if (!valid.valid) {
+      throw new Error(`Failed to validate Module ${this.name} configuration: ${valid.errors.join(', ')}`);
+    }
+  }
+
+  protected getDefaultId(dataSourceName: string): string {
+    return `${this.type}-${dataSourceName}`;
+  }
+}

package/src/replication/replication-index.ts

@@ -1,7 +1,5 @@
+export * from './AbstractReplicationJob.js';
+export * from './AbstractReplicator.js';
 export * from './ErrorRateLimiter.js';
-export * from './PgRelation.js';
-export * from './util.js';
-export * from './WalConnection.js';
-export * from './WalStream.js';
-export * from './WalStreamManager.js';
-export * from './WalStreamRunner.js';
+export * from './ReplicationEngine.js';
+export * from './ReplicationModule.js';

package/src/routes/RouterEngine.ts

@@ -0,0 +1,120 @@
+import { logger } from '@powersync/lib-services-framework';
+
+import * as api from '../api/api-index.js';
+
+import { ADMIN_ROUTES } from './endpoints/admin.js';
+import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
+import { syncStreamReactive } from './endpoints/socket-route.js';
+import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
+import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
+import { SocketRouteGenerator } from './router-socket.js';
+import { RouteDefinition } from './router.js';
+
+export type RouterSetupResponse = {
+  onShutdown: () => Promise<void>;
+};
+
+export type RouterEngineRoutes = {
+  api_routes: RouteDefinition[];
+  stream_routes: RouteDefinition[];
+  socket_routes: SocketRouteGenerator[];
+};
+
+export type RouterSetup = (routes: RouterEngineRoutes) => Promise<RouterSetupResponse>;
+
+/**
+ *  Serves as a registry from which SyncAPIs can be retrieved based on Replication DataSource type
+ *  Initially only one SyncAPI per DataSource type is supported
+ */
+export class RouterEngine {
+  closed: boolean;
+  routes: RouterEngineRoutes;
+
+  protected stopHandlers: Set<() => void>;
+
+  /**
+   * A final cleanup handler to be executed after all stopHandlers
+   */
+  protected cleanupHandler: (() => Promise<void>) | null;
+
+  private api: api.RouteAPI | null;
+
+  constructor() {
+    this.api = null;
+    this.stopHandlers = new Set();
+    this.cleanupHandler = null;
+    this.closed = false;
+
+    // Default routes
+    this.routes = {
+      api_routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES],
+      stream_routes: [...SYNC_STREAM_ROUTES],
+      socket_routes: [syncStreamReactive]
+    };
+  }
+
+  public registerAPI(api: api.RouteAPI) {
+    if (this.api) {
+      logger.warn('A RouteAPI has already been registered. Overriding existing implementation');
+    }
+
+    this.api = api;
+  }
+
+  public getAPI(): api.RouteAPI {
+    if (!this.api) {
+      throw new Error('No RouteAPI adapter has been registered yet.');
+    }
+    return this.api;
+  }
+
+  /**
+   * Starts the router given the configuration provided
+   */
+  async start(setup: RouterSetup) {
+    logger.info('Starting Router Engine...');
+    const { onShutdown } = await setup(this.routes);
+    this.cleanupHandler = onShutdown;
+    logger.info('Successfully started Router Engine.');
+  }
+
+  /**
+   * Runs all stop handlers then final cleanup.
+   */
+  async shutDown() {
+    logger.info('Shutting down Router Engine...');
+    // Close open streams, so that they don't block the server from closing.
+    // Note: This does not work well when streaming requests are queued. In that case, the server still doesn't
+    // close in the 30-second timeout.
+    this.closed = true;
+
+    logger.info(`Closing ${this.stopHandlers.size} streams.`);
+    for (let handler of this.stopHandlers) {
+      handler();
+    }
+
+    logger.info(`Running cleanup.`);
+
+    // Typically closes the server
+    await this.cleanupHandler?.();
+
+    // Close the api handlers
+    await this.api?.shutdown();
+    logger.info('Successfully shut down Router Engine.');
+  }
+
+  /**
+   * Add a stop handler callback to be executed when the router engine is being
+   * shutdown.
+   */
+  addStopHandler(handler: () => void): () => void {
+    if (this.closed) {
+      handler();
+      return () => {};
+    }
+    this.stopHandlers.add(handler);
+    return () => {
+      this.stopHandlers.delete(handler);
+    };
+  }
+}

package/src/routes/auth.ts

@@ -1,9 +1,9 @@
 import * as jose from 'jose';
 
 import * as auth from '../auth/auth-index.js';
+import { ServiceContext } from '../system/ServiceContext.js';
 import * as util from '../util/util-index.js';
 import { BasicRouterRequest, Context, RequestEndpointHandlerPayload } from './router.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
 
 export function endpoint(req: BasicRouterRequest) {
   const protocol = req.headers['x-forwarded-proto'] ?? req.protocol;
@@ -108,7 +108,7 @@ export async function authorizeUser(context: Context, authHeader: string = '') {
     };
   }
 
-  const { context: tokenContext, errors } = await generateContext(context.system, token);
+  const { context: tokenContext, errors } = await generateContext(context.service_context, token);
 
   if (!tokenContext) {
     return {
@@ -121,14 +121,14 @@ export async function authorizeUser(context: Context, authHeader: string = '') {
   return { authorized: true };
 }
 
-export async function generateContext(system: CorePowerSyncSystem, token: string) {
-  const config = system.config;
+export async function generateContext(serviceContext: ServiceContext, token: string) {
+  const { configuration } = serviceContext;
 
   let tokenPayload: auth.JwtPayload;
   try {
-    const maxAge = config.token_max_expiration;
-    tokenPayload = await system.client_keystore.verifyJwt(token, {
-      defaultAudiences: config.jwt_audiences,
+    const maxAge = configuration.token_max_expiration;
+    tokenPayload = await configuration.client_keystore.verifyJwt(token, {
+      defaultAudiences: configuration.jwt_audiences,
       maxAge: maxAge
     });
     return {
@@ -149,9 +149,14 @@ export async function generateContext(system: CorePowerSyncSystem, token: string
  * @deprecated
  */
 export const authDevUser = async (payload: RequestEndpointHandlerPayload) => {
-  const context = payload.context;
+  const {
+    context: {
+      service_context: { configuration }
+    }
+  } = payload;
+
   const token = getTokenFromHeader(payload.request.headers.authorization as string);
-  if (!context.system.config.dev.demo_auth) {
+  if (!configuration.dev.demo_auth) {
     return {
       authorized: false,
       errors: ['Authentication disabled']
@@ -170,7 +175,7 @@ export const authDevUser = async (payload: RequestEndpointHandlerPayload) => {
 
   let tokenPayload: auth.JwtPayload;
   try {
-    tokenPayload = await context.system.dev_client_keystore.verifyJwt(token, {
+    tokenPayload = await configuration.dev_client_keystore.verifyJwt(token, {
       defaultAudiences: audience,
       maxAge: '31d'
     });
@@ -186,8 +191,12 @@ export const authDevUser = async (payload: RequestEndpointHandlerPayload) => {
 };
 
 export const authApi = (payload: RequestEndpointHandlerPayload) => {
-  const context = payload.context;
-  const api_keys = context.system.config.api_tokens;
+  const {
+    context: {
+      service_context: { configuration }
+    }
+  } = payload;
+  const api_keys = configuration.api_tokens;
   if (api_keys.length == 0) {
     return {
       authorized: false,

package/src/routes/configure-fastify.ts

@@ -5,10 +5,11 @@ import * as system from '../system/system-index.js';
 
 import { ADMIN_ROUTES } from './endpoints/admin.js';
 import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
+import { PROBES_ROUTES } from './endpoints/probes.js';
 import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
 import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
 import { createRequestQueueHook, CreateRequestQueueParams } from './hooks.js';
-import { RouteDefinition } from './router.js';
+import { RouteDefinition, RouterServiceContext } from './router.js';
 
 /**
  * A list of route definitions to be registered as endpoints.
@@ -16,7 +17,7 @@ import { RouteDefinition } from './router.js';
  */
 export type RouteRegistrationOptions = {
   routes: RouteDefinition[];
-  queueOptions: CreateRequestQueueParams;
+  queue_options: CreateRequestQueueParams;
 };
 
 /**
@@ -25,25 +26,25 @@ export type RouteRegistrationOptions = {
  */
 export type RouteDefinitions = {
   api?: Partial<RouteRegistrationOptions>;
-  syncStream?: Partial<RouteRegistrationOptions>;
+  sync_stream?: Partial<RouteRegistrationOptions>;
 };
 
 export type FastifyServerConfig = {
-  system: system.CorePowerSyncSystem;
+  service_context: system.ServiceContext;
   routes?: RouteDefinitions;
 };
 
 export const DEFAULT_ROUTE_OPTIONS = {
   api: {
-    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES],
-    queueOptions: {
+    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES, ...PROBES_ROUTES],
+    queue_options: {
       concurrency: 10,
       max_queue_depth: 20
     }
   },
-  syncStream: {
+  sync_stream: {
     routes: [...SYNC_STREAM_ROUTES],
-    queueOptions: {
+    queue_options: {
       concurrency: 200,
       max_queue_depth: 0
     }
@@ -55,7 +56,20 @@ export const DEFAULT_ROUTE_OPTIONS = {
  * concurrency queue limits or override routes.
  */
 export function configureFastifyServer(server: fastify.FastifyInstance, options: FastifyServerConfig) {
-  const { system, routes = DEFAULT_ROUTE_OPTIONS } = options;
+  const { service_context, routes = DEFAULT_ROUTE_OPTIONS } = options;
+
+  const generateContext = async () => {
+    const { routerEngine } = service_context;
+    if (!routerEngine) {
+      throw new Error(`RouterEngine has not been registered`);
+    }
+
+    return {
+      user_id: undefined,
+      service_context: service_context as RouterServiceContext
+    };
+  };
+
   /**
    * Fastify creates an encapsulated context for each `.register` call.
    * Creating a separate context here to separate the concurrency limits for Admin APIs
@@ -63,20 +77,11 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
    * https://github.com/fastify/fastify/blob/main/docs/Reference/Encapsulation.md
    */
   server.register(async function (childContext) {
-    registerFastifyRoutes(
-      childContext,
-      async () => {
-        return {
-          user_id: undefined,
-          system: system
-        };
-      },
-      routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes
-    );
+    registerFastifyRoutes(childContext, generateContext, routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes);
     // Limit the active concurrent requests
     childContext.addHook(
       'onRequest',
-      createRequestQueueHook(routes.api?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.api.queueOptions)
+      createRequestQueueHook(routes.api?.queue_options ?? DEFAULT_ROUTE_OPTIONS.api.queue_options)
     );
   });
 
@@ -84,18 +89,13 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
   server.register(async function (childContext) {
     registerFastifyRoutes(
       childContext,
-      async () => {
-        return {
-          user_id: undefined,
-          system: system
-        };
-      },
-      routes.syncStream?.routes ?? DEFAULT_ROUTE_OPTIONS.syncStream.routes
+      generateContext,
+      routes.sync_stream?.routes ?? DEFAULT_ROUTE_OPTIONS.sync_stream.routes
     );
     // Limit the active concurrent requests
     childContext.addHook(
       'onRequest',
-      createRequestQueueHook(routes.syncStream?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.syncStream.queueOptions)
+      createRequestQueueHook(routes.sync_stream?.queue_options ?? DEFAULT_ROUTE_OPTIONS.sync_stream.queue_options)
     );
   });
 }

package/src/routes/configure-rsocket.ts

@@ -4,22 +4,22 @@ import * as http from 'http';
 import { errors, logger } from '@powersync/lib-services-framework';
 import { ReactiveSocketRouter, RSocketRequestMeta } from '@powersync/service-rsocket-router';
 
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { ServiceContext } from '../system/ServiceContext.js';
 import { generateContext, getTokenFromHeader } from './auth.js';
 import { syncStreamReactive } from './endpoints/socket-route.js';
 import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
-import { Context } from './router.js';
+import { Context, RouterServiceContext } from './router.js';
 
 export type RSockerRouterConfig = {
-  system: CorePowerSyncSystem;
+  service_context: ServiceContext;
   server: http.Server;
-  routeGenerators?: SocketRouteGenerator[];
+  route_generators?: SocketRouteGenerator[];
 };
 
 export const DEFAULT_SOCKET_ROUTES = [syncStreamReactive];
 
 export function configureRSocket(router: ReactiveSocketRouter<Context>, options: RSockerRouterConfig) {
-  const { routeGenerators = DEFAULT_SOCKET_ROUTES, server, system } = options;
+  const { route_generators = DEFAULT_SOCKET_ROUTES, server, service_context } = options;
 
   router.applyWebSocketEndpoints(server, {
     contextProvider: async (data: Buffer) => {
@@ -32,16 +32,21 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
       try {
         const extracted_token = getTokenFromHeader(token);
         if (extracted_token != null) {
-          const { context, errors: token_errors } = await generateContext(system, extracted_token);
+          const { context, errors: token_errors } = await generateContext(options.service_context, extracted_token);
           if (context?.token_payload == null) {
             throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
           }
+
+          if (!service_context.routerEngine) {
+            throw new Error(`RouterEngine has not been registered`);
+          }
+
           return {
             token,
             user_agent,
             ...context,
             token_errors: token_errors,
-            system
+            service_context: service_context as RouterServiceContext
           };
         } else {
           throw new errors.AuthorizationError('No token provided');
@@ -51,7 +56,7 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
         throw ex;
       }
     },
-    endpoints: routeGenerators.map((generator) => generator(router)),
+    endpoints: route_generators.map((generator) => generator(router)),
     metaDecoder: async (meta: Buffer) => {
       return RSocketRequestMeta.decode(deserialize(meta) as any);
     },