@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/src/routes/auth.ts

@@ -1,9 +1,9 @@
 import * as jose from 'jose';
 
 import * as auth from '../auth/auth-index.js';
+import { ServiceContext } from '../system/ServiceContext.js';
 import * as util from '../util/util-index.js';
 import { BasicRouterRequest, Context, RequestEndpointHandlerPayload } from './router.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
 
 export function endpoint(req: BasicRouterRequest) {
   const protocol = req.headers['x-forwarded-proto'] ?? req.protocol;
@@ -108,7 +108,7 @@ export async function authorizeUser(context: Context, authHeader: string = '') {
     };
   }
 
-  const { context: tokenContext, errors } = await generateContext(context.system, token);
+  const { context: tokenContext, errors } = await generateContext(context.service_context, token);
 
   if (!tokenContext) {
     return {
@@ -121,14 +121,14 @@ export async function authorizeUser(context: Context, authHeader: string = '') {
   return { authorized: true };
 }
 
-export async function generateContext(system: CorePowerSyncSystem, token: string) {
-  const config = system.config;
+export async function generateContext(serviceContext: ServiceContext, token: string) {
+  const { configuration } = serviceContext;
 
   let tokenPayload: auth.JwtPayload;
   try {
-    const maxAge = config.token_max_expiration;
-    tokenPayload = await system.client_keystore.verifyJwt(token, {
-      defaultAudiences: config.jwt_audiences,
+    const maxAge = configuration.token_max_expiration;
+    tokenPayload = await configuration.client_keystore.verifyJwt(token, {
+      defaultAudiences: configuration.jwt_audiences,
       maxAge: maxAge
     });
     return {
@@ -149,9 +149,14 @@ export async function generateContext(system: CorePowerSyncSystem, token: string
  * @deprecated
  */
 export const authDevUser = async (payload: RequestEndpointHandlerPayload) => {
-  const context = payload.context;
+  const {
+    context: {
+      service_context: { configuration }
+    }
+  } = payload;
+
   const token = getTokenFromHeader(payload.request.headers.authorization as string);
-  if (!context.system.config.dev.demo_auth) {
+  if (!configuration.dev.demo_auth) {
     return {
       authorized: false,
       errors: ['Authentication disabled']
@@ -170,7 +175,7 @@ export const authDevUser = async (payload: RequestEndpointHandlerPayload) => {
 
   let tokenPayload: auth.JwtPayload;
   try {
-    tokenPayload = await context.system.dev_client_keystore.verifyJwt(token, {
+    tokenPayload = await configuration.dev_client_keystore.verifyJwt(token, {
       defaultAudiences: audience,
       maxAge: '31d'
     });
@@ -186,8 +191,12 @@ export const authDevUser = async (payload: RequestEndpointHandlerPayload) => {
 };
 
 export const authApi = (payload: RequestEndpointHandlerPayload) => {
-  const context = payload.context;
-  const api_keys = context.system.config.api_tokens;
+  const {
+    context: {
+      service_context: { configuration }
+    }
+  } = payload;
+  const api_keys = configuration.api_tokens;
   if (api_keys.length == 0) {
     return {
       authorized: false,

package/src/routes/configure-fastify.ts

@@ -0,0 +1,101 @@
+import type fastify from 'fastify';
+import { registerFastifyRoutes } from './route-register.js';
+
+import * as system from '../system/system-index.js';
+
+import { ADMIN_ROUTES } from './endpoints/admin.js';
+import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
+import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
+import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
+import { createRequestQueueHook, CreateRequestQueueParams } from './hooks.js';
+import { RouteDefinition } from './router.js';
+
+/**
+ * A list of route definitions to be registered as endpoints.
+ * Supplied concurrency limits will be applied to the grouped routes.
+ */
+export type RouteRegistrationOptions = {
+  routes: RouteDefinition[];
+  queue_options: CreateRequestQueueParams;
+};
+
+/**
+ * HTTP routes separated by API and Sync stream categories.
+ * This allows for separate concurrency limits.
+ */
+export type RouteDefinitions = {
+  api?: Partial<RouteRegistrationOptions>;
+  sync_stream?: Partial<RouteRegistrationOptions>;
+};
+
+export type FastifyServerConfig = {
+  service_context: system.ServiceContext;
+  routes?: RouteDefinitions;
+};
+
+export const DEFAULT_ROUTE_OPTIONS = {
+  api: {
+    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES],
+    queue_options: {
+      concurrency: 10,
+      max_queue_depth: 20
+    }
+  },
+  sync_stream: {
+    routes: [...SYNC_STREAM_ROUTES],
+    queue_options: {
+      concurrency: 200,
+      max_queue_depth: 0
+    }
+  }
+};
+
+/**
+ * Registers default routes on a Fastify server. Consumers can optionally configure
+ * concurrency queue limits or override routes.
+ */
+export function configureFastifyServer(server: fastify.FastifyInstance, options: FastifyServerConfig) {
+  const { service_context, routes = DEFAULT_ROUTE_OPTIONS } = options;
+  /**
+   * Fastify creates an encapsulated context for each `.register` call.
+   * Creating a separate context here to separate the concurrency limits for Admin APIs
+   * and Sync Streaming routes.
+   * https://github.com/fastify/fastify/blob/main/docs/Reference/Encapsulation.md
+   */
+  server.register(async function (childContext) {
+    registerFastifyRoutes(
+      childContext,
+      async () => {
+        return {
+          user_id: undefined,
+          service_context
+        };
+      },
+      routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes
+    );
+    // Limit the active concurrent requests
+    childContext.addHook(
+      'onRequest',
+      createRequestQueueHook(routes.api?.queue_options ?? DEFAULT_ROUTE_OPTIONS.api.queue_options)
+    );
+  });
+
+  // Create a separate context for concurrency queueing
+  server.register(async function (childContext) {
+    registerFastifyRoutes(
+      childContext,
+      async () => {
+        return {
+          user_id: undefined,
+          service_context
+        };
+      },
+      routes.sync_stream?.routes ?? DEFAULT_ROUTE_OPTIONS.sync_stream.routes
+    );
+    // Limit the active concurrent requests
+    childContext.addHook(
+      'onRequest',
+      createRequestQueueHook(routes.sync_stream?.queue_options ?? DEFAULT_ROUTE_OPTIONS.sync_stream.queue_options)
+    );
+  });
+}

package/src/routes/configure-rsocket.ts

@@ -0,0 +1,60 @@
+import { deserialize } from 'bson';
+import * as http from 'http';
+
+import { errors, logger } from '@powersync/lib-services-framework';
+import { ReactiveSocketRouter, RSocketRequestMeta } from '@powersync/service-rsocket-router';
+
+import { ServiceContext } from '../system/ServiceContext.js';
+import { generateContext, getTokenFromHeader } from './auth.js';
+import { syncStreamReactive } from './endpoints/socket-route.js';
+import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
+import { Context } from './router.js';
+
+export type RSockerRouterConfig = {
+  service_context: ServiceContext;
+  server: http.Server;
+  route_generators?: SocketRouteGenerator[];
+};
+
+export const DEFAULT_SOCKET_ROUTES = [syncStreamReactive];
+
+export function configureRSocket(router: ReactiveSocketRouter<Context>, options: RSockerRouterConfig) {
+  const { route_generators = DEFAULT_SOCKET_ROUTES, server, service_context } = options;
+
+  router.applyWebSocketEndpoints(server, {
+    contextProvider: async (data: Buffer) => {
+      const { token, user_agent } = RSocketContextMeta.decode(deserialize(data) as any);
+
+      if (!token) {
+        throw new errors.AuthorizationError('No token provided');
+      }
+
+      try {
+        const extracted_token = getTokenFromHeader(token);
+        if (extracted_token != null) {
+          const { context, errors: token_errors } = await generateContext(options.service_context, extracted_token);
+          if (context?.token_payload == null) {
+            throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
+          }
+          return {
+            token,
+            user_agent,
+            ...context,
+            token_errors: token_errors,
+            service_context
+          };
+        } else {
+          throw new errors.AuthorizationError('No token provided');
+        }
+      } catch (ex) {
+        logger.error(ex);
+        throw ex;
+      }
+    },
+    endpoints: route_generators.map((generator) => generator(router)),
+    metaDecoder: async (meta: Buffer) => {
+      return RSocketRequestMeta.decode(deserialize(meta) as any);
+    },
+    payloadDecoder: async (rawData?: Buffer) => rawData && deserialize(rawData)
+  });
+}

package/src/routes/endpoints/admin.ts

@@ -1,35 +1,11 @@
 import { errors, router, schema } from '@powersync/lib-services-framework';
-import { SqlSyncRules, SqliteValue, StaticSchema, isJsonValue, toSyncRulesValue } from '@powersync/service-sync-rules';
+import { SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
 import { internal_routes } from '@powersync/service-types';
 
 import * as api from '../../api/api-index.js';
-import * as util from '../../util/util-index.js';
-
-import { routeDefinition } from '../router.js';
-import { PersistedSyncRulesContent } from '../../storage/BucketStorage.js';
+import * as storage from '../../storage/storage-index.js';
 import { authApi } from '../auth.js';
-
-const demoCredentials = routeDefinition({
-  path: '/api/admin/v1/demo-credentials',
-  method: router.HTTPMethod.POST,
-  authorize: authApi,
-  validator: schema.createTsCodecValidator(internal_routes.DemoCredentialsRequest, {
-    allowAdditional: true
-  }),
-  handler: async (payload) => {
-    const connection = payload.context.system.config.connection;
-    if (connection == null || !connection.demo_database) {
-      return internal_routes.DemoCredentialsResponse.encode({});
-    }
-
-    const uri = util.buildDemoPgUri(connection);
-    return internal_routes.DemoCredentialsResponse.encode({
-      credentials: {
-        postgres_uri: uri
-      }
-    });
-  }
-});
+import { routeDefinition } from '../router.js';
 
 export const executeSql = routeDefinition({
   path: '/api/admin/v1/execute-sql',
@@ -37,47 +13,27 @@ export const executeSql = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ExecuteSqlRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const connection = payload.context.system.config.connection;
-    if (connection == null || !connection.debug_api) {
-      return internal_routes.ExecuteSqlResponse.encode({
-        results: {
-          columns: [],
-          rows: []
-        },
-        success: false,
-        error: 'SQL querying is not enabled'
-      });
-    }
-
-    const pool = payload.context.system.requirePgPool();
-
-    const { query, args } = payload.params.sql;
+    const {
+      params: {
+        sql: { query, args }
+      }
+    } = payload;
 
-    try {
-      const result = await pool.query({
-        statement: query,
-        params: args.map(util.autoParameter)
-      });
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-      return internal_routes.ExecuteSqlResponse.encode({
-        success: true,
-        results: {
-          columns: result.columns.map((c) => c.name),
-          rows: result.rows.map((row) => {
-            return row.map((value) => mapColumnValue(toSyncRulesValue(value)));
-          })
-        }
-      });
-    } catch (e) {
+    const sourceConfig = await apiHandler.getSourceConfig();
+    if (!sourceConfig.debug_api) {
       return internal_routes.ExecuteSqlResponse.encode({
         results: {
           columns: [],
           rows: []
         },
         success: false,
-        error: e.message
+        error: 'SQL querying is not enabled'
       });
     }
+
+    return internal_routes.ExecuteSqlResponse.encode(await apiHandler.executeQuery(query, args));
   }
 });
 
@@ -87,34 +43,45 @@ export const diagnostics = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.DiagnosticsRequest, { allowAdditional: true }),
   handler: async (payload) => {
+    const { context } = payload;
+    const { service_context } = context;
     const include_content = payload.params.sync_rules_content ?? false;
-    const system = payload.context.system;
 
-    const status = await api.getConnectionStatus(system);
-    if (status == null) {
+    const apiHandler = service_context.routerEngine!.getAPI();
+
+    const status = await apiHandler.getConnectionStatus();
+    if (!status) {
       return internal_routes.DiagnosticsResponse.encode({
         connections: []
       });
     }
 
-    const { storage } = system;
-    const active = await storage.getActiveSyncRulesContent();
-    const next = await storage.getNextSyncRulesContent();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const active = await activeBucketStorage.getActiveSyncRulesContent();
+    const next = await activeBucketStorage.getNextSyncRulesContent();
 
-    const active_status = await api.getSyncRulesStatus(active, system, {
+    const active_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, active, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
-    const next_status = await api.getSyncRulesStatus(next, system, {
+    const next_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, next, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
     return internal_routes.DiagnosticsResponse.encode({
-      connections: [status],
+      connections: [
+        {
+          ...status,
+          // TODO update this in future
+          postgres_uri: status.uri
+        }
+      ],
       active_sync_rules: active_status,
       deploying_sync_rules: next_status
     });
@@ -127,9 +94,9 @@ export const getSchema = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.GetSchemaRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(system));
+    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(apiHandler));
   }
 });
 
@@ -139,15 +106,19 @@ export const reprocess = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ReprocessRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
-    const storage = system.storage;
-    const next = await storage.getNextSyncRules();
+    const {
+      context: { service_context }
+    } = payload;
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const apiHandler = service_context.routerEngine!.getAPI();
+    const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
     if (next != null) {
       throw new Error(`Busy processing sync rules - cannot reprocess`);
     }
 
-    const active = await storage.getActiveSyncRules();
+    const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
     if (active == null) {
       throw new errors.JourneyError({
         status: 422,
@@ -156,15 +127,18 @@ export const reprocess = routeDefinition({
       });
     }
 
-    const new_rules = await storage.updateSyncRules({
+    const new_rules = await activeBucketStorage.updateSyncRules({
       content: active.sync_rules.content
     });
 
+    const baseConfig = await apiHandler.getSourceConfig();
+
     return internal_routes.ReprocessResponse.encode({
       connections: [
         {
-          tag: system.config.connection!.tag,
-          id: system.config.connection!.id,
+          // Previously the connection was asserted with `!`
+          tag: baseConfig!.tag!,
+          id: baseConfig!.id,
           slot_name: new_rules.slot_name
         }
       ]
@@ -178,14 +152,16 @@ export const validate = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ValidateRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
+    const {
+      context: { service_context }
+    } = payload;
     const content = payload.params.sync_rules;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const schemaData = await api.getConnectionsSchema(system);
+    const schemaData = await api.getConnectionsSchema(apiHandler);
     const schema = new StaticSchema(schemaData.connections);
 
-    const sync_rules: PersistedSyncRulesContent = {
+    const sync_rules: storage.PersistedSyncRulesContent = {
       // Dummy values
       id: 0,
       slot_name: '',
@@ -193,7 +169,10 @@ export const validate = routeDefinition({
       parsed() {
         return {
           ...this,
-          sync_rules: SqlSyncRules.fromYaml(content, { throwOnError: false, schema })
+          sync_rules: SqlSyncRules.fromYaml(content, {
+            ...apiHandler.getParseSyncRulesOptions(),
+            schema
+          })
         };
       },
       sync_rules_content: content,
@@ -202,19 +181,24 @@ export const validate = routeDefinition({
       }
     };
 
-    const connectionStatus = await api.getConnectionStatus(system);
-    if (connectionStatus == null) {
+    const connectionStatus = await apiHandler.getConnectionStatus();
+    if (!connectionStatus) {
       return internal_routes.ValidateResponse.encode({
         errors: [{ level: 'fatal', message: 'No connection configured' }],
         connections: []
       });
     }
 
-    const status = (await api.getSyncRulesStatus(sync_rules, system, {
-      include_content: false,
-      check_connection: connectionStatus?.connected,
-      live_status: false
-    }))!;
+    const status = (await api.getSyncRulesStatus(
+      service_context.storageEngine.activeBucketStorage,
+      apiHandler,
+      sync_rules,
+      {
+        include_content: false,
+        check_connection: connectionStatus.connected,
+        live_status: false
+      }
+    ))!;
 
     if (connectionStatus == null) {
       status.errors.push({ level: 'fatal', message: 'No connection configured' });
@@ -224,14 +208,4 @@ export const validate = routeDefinition({
   }
 });
 
-function mapColumnValue(value: SqliteValue) {
-  if (typeof value == 'bigint') {
-    return Number(value);
-  } else if (isJsonValue(value)) {
-    return value;
-  } else {
-    return null;
-  }
-}
-
-export const ADMIN_ROUTES = [demoCredentials, executeSql, diagnostics, getSchema, reprocess, validate];
+export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate];

package/src/routes/endpoints/checkpointing.ts

@@ -1,11 +1,13 @@
 import * as t from 'ts-codec';
-import { router, schema } from '@powersync/lib-services-framework';
+import { logger, router, schema } from '@powersync/lib-services-framework';
 
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
-const WriteCheckpointRequest = t.object({});
+const WriteCheckpointRequest = t.object({
+  client_id: t.string.optional()
+});
 
 export const writeCheckpoint = routeDefinition({
   path: '/write-checkpoint.json',
@@ -13,13 +15,33 @@ export const writeCheckpoint = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-    const storage = system.storage;
+    const {
+      context: { service_context }
+    } = payload;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
-    return {
-      checkpoint
-    };
+    // This old API needs a persisted checkpoint id.
+    // Since we don't use LSNs anymore, the only way to get that is to wait.
+    const start = Date.now();
+
+    const head = await apiHandler.getReplicationHead();
+
+    const timeout = 50_000;
+
+    logger.info(`Waiting for LSN checkpoint: ${head}`);
+    while (Date.now() - start < timeout) {
+      const cp = await service_context.storageEngine.activeBucketStorage.getActiveCheckpoint();
+      if (!cp.hasSyncRules()) {
+        throw new Error('No sync rules available');
+      }
+      if (cp.lsn && cp.lsn >= head) {
+        logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);
+        return { checkpoint: cp.checkpoint };
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 30));
+    }
+    throw new Error('Timeout while waiting for checkpoint');
   }
 });
 
@@ -29,11 +51,23 @@ export const writeCheckpoint2 = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const { user_id, system } = payload.context;
-    const storage = system.storage;
-    const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, user_id!);
+    const { user_id, service_context } = payload.context;
+
+    const apiHandler = service_context.routerEngine!.getAPI();
+
+    const client_id = payload.params.client_id;
+    const full_user_id = util.checkpointUserId(user_id, client_id);
+
+    const currentCheckpoint = await apiHandler.getReplicationHead();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+
+    const writeCheckpoint = await activeBucketStorage.createWriteCheckpoint(full_user_id, { '1': currentCheckpoint });
+    logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`);
+
     return {
-      write_checkpoint: String(write_checkpoint)
+      write_checkpoint: String(writeCheckpoint)
     };
   }
 });

package/src/routes/endpoints/route-endpoints-index.ts

@@ -1,6 +1,5 @@
 export * from './admin.js';
 export * from './checkpointing.js';
-export * from './dev.js';
 export * from './socket-route.js';
 export * from './sync-rules.js';
 export * from './sync-stream.js';