@powersync/service-core 0.8.8 → 0.10.0

package/src/routes/configure-fastify.ts

@@ -9,7 +9,7 @@ import { PROBES_ROUTES } from './endpoints/probes.js';
 import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
 import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
 import { createRequestQueueHook, CreateRequestQueueParams } from './hooks.js';
-import { RouteDefinition } from './router.js';
+import { RouteDefinition, RouterServiceContext } from './router.js';
 
 /**
  * A list of route definitions to be registered as endpoints.
@@ -17,7 +17,7 @@ import { RouteDefinition } from './router.js';
  */
 export type RouteRegistrationOptions = {
   routes: RouteDefinition[];
-  queueOptions: CreateRequestQueueParams;
+  queue_options: CreateRequestQueueParams;
 };
 
 /**
@@ -26,25 +26,25 @@ export type RouteRegistrationOptions = {
  */
 export type RouteDefinitions = {
   api?: Partial<RouteRegistrationOptions>;
-  syncStream?: Partial<RouteRegistrationOptions>;
+  sync_stream?: Partial<RouteRegistrationOptions>;
 };
 
 export type FastifyServerConfig = {
-  system: system.CorePowerSyncSystem;
+  service_context: system.ServiceContext;
   routes?: RouteDefinitions;
 };
 
 export const DEFAULT_ROUTE_OPTIONS = {
   api: {
     routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES, ...PROBES_ROUTES],
-    queueOptions: {
+    queue_options: {
       concurrency: 10,
       max_queue_depth: 20
     }
   },
-  syncStream: {
+  sync_stream: {
     routes: [...SYNC_STREAM_ROUTES],
-    queueOptions: {
+    queue_options: {
       concurrency: 200,
       max_queue_depth: 0
     }
@@ -56,7 +56,20 @@ export const DEFAULT_ROUTE_OPTIONS = {
  * concurrency queue limits or override routes.
  */
 export function configureFastifyServer(server: fastify.FastifyInstance, options: FastifyServerConfig) {
-  const { system, routes = DEFAULT_ROUTE_OPTIONS } = options;
+  const { service_context, routes = DEFAULT_ROUTE_OPTIONS } = options;
+
+  const generateContext = async () => {
+    const { routerEngine } = service_context;
+    if (!routerEngine) {
+      throw new Error(`RouterEngine has not been registered`);
+    }
+
+    return {
+      user_id: undefined,
+      service_context: service_context as RouterServiceContext
+    };
+  };
+
   /**
    * Fastify creates an encapsulated context for each `.register` call.
    * Creating a separate context here to separate the concurrency limits for Admin APIs
@@ -64,20 +77,11 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
    * https://github.com/fastify/fastify/blob/main/docs/Reference/Encapsulation.md
    */
   server.register(async function (childContext) {
-    registerFastifyRoutes(
-      childContext,
-      async () => {
-        return {
-          user_id: undefined,
-          system: system
-        };
-      },
-      routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes
-    );
+    registerFastifyRoutes(childContext, generateContext, routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes);
     // Limit the active concurrent requests
     childContext.addHook(
       'onRequest',
-      createRequestQueueHook(routes.api?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.api.queueOptions)
+      createRequestQueueHook(routes.api?.queue_options ?? DEFAULT_ROUTE_OPTIONS.api.queue_options)
     );
   });
 
@@ -85,18 +89,13 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
   server.register(async function (childContext) {
     registerFastifyRoutes(
       childContext,
-      async () => {
-        return {
-          user_id: undefined,
-          system: system
-        };
-      },
-      routes.syncStream?.routes ?? DEFAULT_ROUTE_OPTIONS.syncStream.routes
+      generateContext,
+      routes.sync_stream?.routes ?? DEFAULT_ROUTE_OPTIONS.sync_stream.routes
     );
     // Limit the active concurrent requests
     childContext.addHook(
       'onRequest',
-      createRequestQueueHook(routes.syncStream?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.syncStream.queueOptions)
+      createRequestQueueHook(routes.sync_stream?.queue_options ?? DEFAULT_ROUTE_OPTIONS.sync_stream.queue_options)
     );
   });
 }

package/src/routes/configure-rsocket.ts

@@ -4,22 +4,22 @@ import * as http from 'http';
 import { errors, logger } from '@powersync/lib-services-framework';
 import { ReactiveSocketRouter, RSocketRequestMeta } from '@powersync/service-rsocket-router';
 
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { ServiceContext } from '../system/ServiceContext.js';
 import { generateContext, getTokenFromHeader } from './auth.js';
 import { syncStreamReactive } from './endpoints/socket-route.js';
 import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
-import { Context } from './router.js';
+import { Context, RouterServiceContext } from './router.js';
 
 export type RSockerRouterConfig = {
-  system: CorePowerSyncSystem;
+  service_context: ServiceContext;
   server: http.Server;
-  routeGenerators?: SocketRouteGenerator[];
+  route_generators?: SocketRouteGenerator[];
 };
 
 export const DEFAULT_SOCKET_ROUTES = [syncStreamReactive];
 
 export function configureRSocket(router: ReactiveSocketRouter<Context>, options: RSockerRouterConfig) {
-  const { routeGenerators = DEFAULT_SOCKET_ROUTES, server, system } = options;
+  const { route_generators = DEFAULT_SOCKET_ROUTES, server, service_context } = options;
 
   router.applyWebSocketEndpoints(server, {
     contextProvider: async (data: Buffer) => {
@@ -32,16 +32,21 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
       try {
         const extracted_token = getTokenFromHeader(token);
         if (extracted_token != null) {
-          const { context, errors: token_errors } = await generateContext(system, extracted_token);
+          const { context, errors: token_errors } = await generateContext(options.service_context, extracted_token);
           if (context?.token_payload == null) {
             throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
           }
+
+          if (!service_context.routerEngine) {
+            throw new Error(`RouterEngine has not been registered`);
+          }
+
           return {
             token,
             user_agent,
             ...context,
             token_errors: token_errors,
-            system
+            service_context: service_context as RouterServiceContext
           };
         } else {
           throw new errors.AuthorizationError('No token provided');
@@ -51,7 +56,7 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
         throw ex;
       }
     },
-    endpoints: routeGenerators.map((generator) => generator(router)),
+    endpoints: route_generators.map((generator) => generator(router)),
     metaDecoder: async (meta: Buffer) => {
       return RSocketRequestMeta.decode(deserialize(meta) as any);
     },

package/src/routes/endpoints/admin.ts

@@ -1,11 +1,9 @@
 import { errors, router, schema } from '@powersync/lib-services-framework';
-import { SqlSyncRules, SqliteValue, StaticSchema, isJsonValue, toSyncRulesValue } from '@powersync/service-sync-rules';
+import { SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
 import { internal_routes } from '@powersync/service-types';
 
 import * as api from '../../api/api-index.js';
-import * as util from '../../util/util-index.js';
-
-import { PersistedSyncRulesContent } from '../../storage/BucketStorage.js';
+import * as storage from '../../storage/storage-index.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
@@ -15,47 +13,27 @@ export const executeSql = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ExecuteSqlRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const connection = payload.context.system.config.connection;
-    if (connection == null || !connection.debug_api) {
-      return internal_routes.ExecuteSqlResponse.encode({
-        results: {
-          columns: [],
-          rows: []
-        },
-        success: false,
-        error: 'SQL querying is not enabled'
-      });
-    }
-
-    const pool = payload.context.system.requirePgPool();
-
-    const { query, args } = payload.params.sql;
+    const {
+      params: {
+        sql: { query, args }
+      }
+    } = payload;
 
-    try {
-      const result = await pool.query({
-        statement: query,
-        params: args.map(util.autoParameter)
-      });
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-      return internal_routes.ExecuteSqlResponse.encode({
-        success: true,
-        results: {
-          columns: result.columns.map((c) => c.name),
-          rows: result.rows.map((row) => {
-            return row.map((value) => mapColumnValue(toSyncRulesValue(value)));
-          })
-        }
-      });
-    } catch (e) {
+    const sourceConfig = await apiHandler.getSourceConfig();
+    if (!sourceConfig.debug_api) {
       return internal_routes.ExecuteSqlResponse.encode({
         results: {
           columns: [],
           rows: []
         },
         success: false,
-        error: e.message
+        error: 'SQL querying is not enabled'
       });
     }
+
+    return internal_routes.ExecuteSqlResponse.encode(await apiHandler.executeQuery(query, args));
   }
 });
 
@@ -65,34 +43,45 @@ export const diagnostics = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.DiagnosticsRequest, { allowAdditional: true }),
   handler: async (payload) => {
+    const { context } = payload;
+    const { service_context } = context;
     const include_content = payload.params.sync_rules_content ?? false;
-    const system = payload.context.system;
 
-    const status = await api.getConnectionStatus(system);
-    if (status == null) {
+    const apiHandler = service_context.routerEngine!.getAPI();
+
+    const status = await apiHandler.getConnectionStatus();
+    if (!status) {
       return internal_routes.DiagnosticsResponse.encode({
         connections: []
       });
     }
 
-    const { storage } = system;
-    const active = await storage.getActiveSyncRulesContent();
-    const next = await storage.getNextSyncRulesContent();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const active = await activeBucketStorage.getActiveSyncRulesContent();
+    const next = await activeBucketStorage.getNextSyncRulesContent();
 
-    const active_status = await api.getSyncRulesStatus(active, system, {
+    const active_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, active, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
-    const next_status = await api.getSyncRulesStatus(next, system, {
+    const next_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, next, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
     return internal_routes.DiagnosticsResponse.encode({
-      connections: [status],
+      connections: [
+        {
+          ...status,
+          // TODO update this in future
+          postgres_uri: status.uri
+        }
+      ],
       active_sync_rules: active_status,
       deploying_sync_rules: next_status
     });
@@ -105,9 +94,9 @@ export const getSchema = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.GetSchemaRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(system));
+    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(apiHandler));
   }
 });
 
@@ -117,15 +106,19 @@ export const reprocess = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ReprocessRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
-    const storage = system.storage;
-    const next = await storage.getNextSyncRules();
+    const {
+      context: { service_context }
+    } = payload;
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const apiHandler = service_context.routerEngine!.getAPI();
+    const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
     if (next != null) {
       throw new Error(`Busy processing sync rules - cannot reprocess`);
     }
 
-    const active = await storage.getActiveSyncRules();
+    const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
     if (active == null) {
       throw new errors.JourneyError({
         status: 422,
@@ -134,15 +127,18 @@ export const reprocess = routeDefinition({
       });
     }
 
-    const new_rules = await storage.updateSyncRules({
+    const new_rules = await activeBucketStorage.updateSyncRules({
       content: active.sync_rules.content
     });
 
+    const baseConfig = await apiHandler.getSourceConfig();
+
     return internal_routes.ReprocessResponse.encode({
       connections: [
         {
-          tag: system.config.connection!.tag,
-          id: system.config.connection!.id,
+          // Previously the connection was asserted with `!`
+          tag: baseConfig.tag,
+          id: baseConfig.id,
           slot_name: new_rules.slot_name
         }
       ]
@@ -156,14 +152,16 @@ export const validate = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ValidateRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
+    const {
+      context: { service_context }
+    } = payload;
     const content = payload.params.sync_rules;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const schemaData = await api.getConnectionsSchema(system);
+    const schemaData = await api.getConnectionsSchema(apiHandler);
     const schema = new StaticSchema(schemaData.connections);
 
-    const sync_rules: PersistedSyncRulesContent = {
+    const sync_rules: storage.PersistedSyncRulesContent = {
       // Dummy values
       id: 0,
       slot_name: '',
@@ -171,7 +169,10 @@ export const validate = routeDefinition({
       parsed() {
         return {
           ...this,
-          sync_rules: SqlSyncRules.fromYaml(content, { throwOnError: false, schema })
+          sync_rules: SqlSyncRules.fromYaml(content, {
+            ...apiHandler.getParseSyncRulesOptions(),
+            schema
+          })
         };
       },
       sync_rules_content: content,
@@ -180,19 +181,24 @@ export const validate = routeDefinition({
       }
     };
 
-    const connectionStatus = await api.getConnectionStatus(system);
-    if (connectionStatus == null) {
+    const connectionStatus = await apiHandler.getConnectionStatus();
+    if (!connectionStatus) {
       return internal_routes.ValidateResponse.encode({
         errors: [{ level: 'fatal', message: 'No connection configured' }],
         connections: []
       });
     }
 
-    const status = (await api.getSyncRulesStatus(sync_rules, system, {
-      include_content: false,
-      check_connection: connectionStatus?.connected,
-      live_status: false
-    }))!;
+    const status = (await api.getSyncRulesStatus(
+      service_context.storageEngine.activeBucketStorage,
+      apiHandler,
+      sync_rules,
+      {
+        include_content: false,
+        check_connection: connectionStatus.connected,
+        live_status: false
+      }
+    ))!;
 
     if (connectionStatus == null) {
       status.errors.push({ level: 'fatal', message: 'No connection configured' });
@@ -202,14 +208,4 @@ export const validate = routeDefinition({
   }
 });
 
-function mapColumnValue(value: SqliteValue) {
-  if (typeof value == 'bigint') {
-    return Number(value);
-  } else if (isJsonValue(value)) {
-    return value;
-  } else {
-    return null;
-  }
-}
-
 export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate];

package/src/routes/endpoints/checkpointing.ts

@@ -1,6 +1,7 @@
+import { logger, router, schema } from '@powersync/lib-services-framework';
 import * as t from 'ts-codec';
-import { router, schema } from '@powersync/lib-services-framework';
 
+import * as framework from '@powersync/lib-services-framework';
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
@@ -15,13 +16,33 @@ export const writeCheckpoint = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-    const storage = system.storage;
+    const {
+      context: { service_context }
+    } = payload;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
-    return {
-      checkpoint
-    };
+    // This old API needs a persisted checkpoint id.
+    // Since we don't use LSNs anymore, the only way to get that is to wait.
+    const start = Date.now();
+
+    const head = await apiHandler.getReplicationHead();
+
+    const timeout = 50_000;
+
+    logger.info(`Waiting for LSN checkpoint: ${head}`);
+    while (Date.now() - start < timeout) {
+      const cp = await service_context.storageEngine.activeBucketStorage.getActiveCheckpoint();
+      if (!cp.hasSyncRules()) {
+        throw new Error('No sync rules available');
+      }
+      if (cp.lsn && cp.lsn >= head) {
+        logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);
+        return { checkpoint: cp.checkpoint };
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 30));
+    }
+    throw new Error('Timeout while waiting for checkpoint');
   }
 });
 
@@ -31,13 +52,32 @@ export const writeCheckpoint2 = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const { user_id, system } = payload.context;
+    const { user_id, service_context } = payload.context;
+
+    const apiHandler = service_context.routerEngine!.getAPI();
+
     const client_id = payload.params.client_id;
     const full_user_id = util.checkpointUserId(user_id, client_id);
-    const storage = system.storage;
-    const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, full_user_id);
+
+    const currentCheckpoint = await apiHandler.getReplicationHead();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+
+    const activeSyncRules = await activeBucketStorage.getActiveSyncRulesContent();
+    if (!activeSyncRules) {
+      throw new framework.errors.ValidationError(`Cannot create Write Checkpoint since no sync rules are active.`);
+    }
+
+    using syncBucketStorage = activeBucketStorage.getInstance(activeSyncRules);
+    const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
+      user_id: full_user_id,
+      heads: { '1': currentCheckpoint }
+    });
+    logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`);
+
     return {
-      write_checkpoint: String(write_checkpoint)
+      write_checkpoint: String(writeCheckpoint)
     };
   }
 });

package/src/routes/endpoints/socket-route.ts

@@ -12,7 +12,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
     handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
-      const { system } = context;
+      const { service_context } = context;
+      const { routerEngine } = service_context;
 
       // Create our own controller that we can abort directly
       const controller = new AbortController();
@@ -31,7 +32,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         }
       });
 
-      if (system.closed) {
+      if (routerEngine!.closed) {
         responder.onError(
           new errors.JourneyError({
             status: 503,
@@ -45,9 +46,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
 
       const syncParams = new RequestParameters(context.token_payload!, params.parameters ?? {});
 
-      const storage = system.storage;
+      const {
+        storageEngine: { activeBucketStorage }
+      } = service_context;
       // Sanity check before we start the stream
-      const cp = await storage.getActiveCheckpoint();
+      const cp = await activeBucketStorage.getActiveCheckpoint();
       if (!cp.hasSyncRules()) {
         responder.onError(
           new errors.JourneyError({
@@ -60,7 +63,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
-      const removeStopHandler = system.addStopHandler(() => {
+      const removeStopHandler = routerEngine!.addStopHandler(() => {
         controller.abort();
       });
 
@@ -68,7 +71,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       const tracker = new sync.RequestTracker();
       try {
         for await (const data of sync.streamResponse({
-          storage,
+          storage: activeBucketStorage,
+          parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
           params: {
             ...params,
             binary_data: true // always true for web sockets