@powersync/service-core 0.0.0-dev-20240725112650 → 0.0.0-dev-20240918092408

package/src/routes/configure-fastify.ts

@@ -5,7 +5,6 @@ import * as system from '../system/system-index.js';
 
 import { ADMIN_ROUTES } from './endpoints/admin.js';
 import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
-import { DEV_ROUTES } from './endpoints/dev.js';
 import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
 import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
 import { createRequestQueueHook, CreateRequestQueueParams } from './hooks.js';
@@ -17,7 +16,7 @@ import { RouteDefinition } from './router.js';
  */
 export type RouteRegistrationOptions = {
   routes: RouteDefinition[];
-  queueOptions: CreateRequestQueueParams;
+  queue_options: CreateRequestQueueParams;
 };
 
 /**
@@ -26,25 +25,25 @@ export type RouteRegistrationOptions = {
  */
 export type RouteDefinitions = {
   api?: Partial<RouteRegistrationOptions>;
-  syncStream?: Partial<RouteRegistrationOptions>;
+  sync_stream?: Partial<RouteRegistrationOptions>;
 };
 
 export type FastifyServerConfig = {
-  system: system.CorePowerSyncSystem;
+  service_context: system.ServiceContext;
   routes?: RouteDefinitions;
 };
 
 export const DEFAULT_ROUTE_OPTIONS = {
   api: {
-    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...DEV_ROUTES, ...SYNC_RULES_ROUTES],
-    queueOptions: {
+    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES],
+    queue_options: {
       concurrency: 10,
       max_queue_depth: 20
     }
   },
-  syncStream: {
+  sync_stream: {
     routes: [...SYNC_STREAM_ROUTES],
-    queueOptions: {
+    queue_options: {
       concurrency: 200,
       max_queue_depth: 0
     }
@@ -56,7 +55,7 @@ export const DEFAULT_ROUTE_OPTIONS = {
  * concurrency queue limits or override routes.
  */
 export function configureFastifyServer(server: fastify.FastifyInstance, options: FastifyServerConfig) {
-  const { system, routes = DEFAULT_ROUTE_OPTIONS } = options;
+  const { service_context, routes = DEFAULT_ROUTE_OPTIONS } = options;
   /**
    * Fastify creates an encapsulated context for each `.register` call.
    * Creating a separate context here to separate the concurrency limits for Admin APIs
@@ -69,7 +68,7 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
       async () => {
         return {
           user_id: undefined,
-          system: system
+          service_context
         };
       },
       routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes
@@ -77,7 +76,7 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
     // Limit the active concurrent requests
     childContext.addHook(
       'onRequest',
-      createRequestQueueHook(routes.api?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.api.queueOptions)
+      createRequestQueueHook(routes.api?.queue_options ?? DEFAULT_ROUTE_OPTIONS.api.queue_options)
     );
   });
 
@@ -88,15 +87,15 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
       async () => {
         return {
           user_id: undefined,
-          system: system
+          service_context
         };
       },
-      routes.syncStream?.routes ?? DEFAULT_ROUTE_OPTIONS.syncStream.routes
+      routes.sync_stream?.routes ?? DEFAULT_ROUTE_OPTIONS.sync_stream.routes
     );
     // Limit the active concurrent requests
     childContext.addHook(
       'onRequest',
-      createRequestQueueHook(routes.syncStream?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.syncStream.queueOptions)
+      createRequestQueueHook(routes.sync_stream?.queue_options ?? DEFAULT_ROUTE_OPTIONS.sync_stream.queue_options)
     );
   });
 }

package/src/routes/configure-rsocket.ts

@@ -4,26 +4,26 @@ import * as http from 'http';
 import { errors, logger } from '@powersync/lib-services-framework';
 import { ReactiveSocketRouter, RSocketRequestMeta } from '@powersync/service-rsocket-router';
 
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { ServiceContext } from '../system/ServiceContext.js';
 import { generateContext, getTokenFromHeader } from './auth.js';
 import { syncStreamReactive } from './endpoints/socket-route.js';
 import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
 import { Context } from './router.js';
 
 export type RSockerRouterConfig = {
-  system: CorePowerSyncSystem;
+  service_context: ServiceContext;
   server: http.Server;
-  routeGenerators?: SocketRouteGenerator[];
+  route_generators?: SocketRouteGenerator[];
 };
 
 export const DEFAULT_SOCKET_ROUTES = [syncStreamReactive];
 
 export function configureRSocket(router: ReactiveSocketRouter<Context>, options: RSockerRouterConfig) {
-  const { routeGenerators = DEFAULT_SOCKET_ROUTES, server, system } = options;
+  const { route_generators = DEFAULT_SOCKET_ROUTES, server, service_context } = options;
 
   router.applyWebSocketEndpoints(server, {
     contextProvider: async (data: Buffer) => {
-      const { token } = RSocketContextMeta.decode(deserialize(data) as any);
+      const { token, user_agent } = RSocketContextMeta.decode(deserialize(data) as any);
 
       if (!token) {
         throw new errors.AuthorizationError('No token provided');
@@ -32,15 +32,16 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
       try {
         const extracted_token = getTokenFromHeader(token);
         if (extracted_token != null) {
-          const { context, errors: token_errors } = await generateContext(system, extracted_token);
+          const { context, errors: token_errors } = await generateContext(options.service_context, extracted_token);
           if (context?.token_payload == null) {
             throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
           }
           return {
             token,
+            user_agent,
             ...context,
             token_errors: token_errors,
-            system
+            service_context
           };
         } else {
           throw new errors.AuthorizationError('No token provided');
@@ -50,7 +51,7 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
         throw ex;
       }
     },
-    endpoints: routeGenerators.map((generator) => generator(router)),
+    endpoints: route_generators.map((generator) => generator(router)),
     metaDecoder: async (meta: Buffer) => {
       return RSocketRequestMeta.decode(deserialize(meta) as any);
     },

package/src/routes/endpoints/admin.ts

@@ -1,35 +1,11 @@
 import { errors, router, schema } from '@powersync/lib-services-framework';
-import { SqlSyncRules, SqliteValue, StaticSchema, isJsonValue, toSyncRulesValue } from '@powersync/service-sync-rules';
+import { SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
 import { internal_routes } from '@powersync/service-types';
 
 import * as api from '../../api/api-index.js';
-import * as util from '../../util/util-index.js';
-
-import { routeDefinition } from '../router.js';
-import { PersistedSyncRulesContent } from '../../storage/BucketStorage.js';
+import * as storage from '../../storage/storage-index.js';
 import { authApi } from '../auth.js';
-
-const demoCredentials = routeDefinition({
-  path: '/api/admin/v1/demo-credentials',
-  method: router.HTTPMethod.POST,
-  authorize: authApi,
-  validator: schema.createTsCodecValidator(internal_routes.DemoCredentialsRequest, {
-    allowAdditional: true
-  }),
-  handler: async (payload) => {
-    const connection = payload.context.system.config.connection;
-    if (connection == null || !connection.demo_database) {
-      return internal_routes.DemoCredentialsResponse.encode({});
-    }
-
-    const uri = util.buildDemoPgUri(connection);
-    return internal_routes.DemoCredentialsResponse.encode({
-      credentials: {
-        postgres_uri: uri
-      }
-    });
-  }
-});
+import { routeDefinition } from '../router.js';
 
 export const executeSql = routeDefinition({
   path: '/api/admin/v1/execute-sql',
@@ -37,47 +13,27 @@ export const executeSql = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ExecuteSqlRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const connection = payload.context.system.config.connection;
-    if (connection == null || !connection.debug_api) {
-      return internal_routes.ExecuteSqlResponse.encode({
-        results: {
-          columns: [],
-          rows: []
-        },
-        success: false,
-        error: 'SQL querying is not enabled'
-      });
-    }
-
-    const pool = payload.context.system.requirePgPool();
-
-    const { query, args } = payload.params.sql;
+    const {
+      params: {
+        sql: { query, args }
+      }
+    } = payload;
 
-    try {
-      const result = await pool.query({
-        statement: query,
-        params: args.map(util.autoParameter)
-      });
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-      return internal_routes.ExecuteSqlResponse.encode({
-        success: true,
-        results: {
-          columns: result.columns.map((c) => c.name),
-          rows: result.rows.map((row) => {
-            return row.map((value) => mapColumnValue(toSyncRulesValue(value)));
-          })
-        }
-      });
-    } catch (e) {
+    const sourceConfig = await apiHandler.getSourceConfig();
+    if (!sourceConfig.debug_api) {
       return internal_routes.ExecuteSqlResponse.encode({
         results: {
           columns: [],
           rows: []
         },
         success: false,
-        error: e.message
+        error: 'SQL querying is not enabled'
       });
     }
+
+    return internal_routes.ExecuteSqlResponse.encode(await apiHandler.executeQuery(query, args));
   }
 });
 
@@ -87,34 +43,45 @@ export const diagnostics = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.DiagnosticsRequest, { allowAdditional: true }),
   handler: async (payload) => {
+    const { context } = payload;
+    const { service_context } = context;
     const include_content = payload.params.sync_rules_content ?? false;
-    const system = payload.context.system;
 
-    const status = await api.getConnectionStatus(system);
-    if (status == null) {
+    const apiHandler = service_context.routerEngine!.getAPI();
+
+    const status = await apiHandler.getConnectionStatus();
+    if (!status) {
       return internal_routes.DiagnosticsResponse.encode({
         connections: []
       });
     }
 
-    const { storage } = system;
-    const active = await storage.getActiveSyncRulesContent();
-    const next = await storage.getNextSyncRulesContent();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const active = await activeBucketStorage.getActiveSyncRulesContent();
+    const next = await activeBucketStorage.getNextSyncRulesContent();
 
-    const active_status = await api.getSyncRulesStatus(active, system, {
+    const active_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, active, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
-    const next_status = await api.getSyncRulesStatus(next, system, {
+    const next_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, next, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
     return internal_routes.DiagnosticsResponse.encode({
-      connections: [status],
+      connections: [
+        {
+          ...status,
+          // TODO update this in future
+          postgres_uri: status.uri
+        }
+      ],
       active_sync_rules: active_status,
       deploying_sync_rules: next_status
     });
@@ -127,9 +94,9 @@ export const getSchema = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.GetSchemaRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(system));
+    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(apiHandler));
   }
 });
 
@@ -139,15 +106,19 @@ export const reprocess = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ReprocessRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
-    const storage = system.storage;
-    const next = await storage.getNextSyncRules();
+    const {
+      context: { service_context }
+    } = payload;
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const apiHandler = service_context.routerEngine!.getAPI();
+    const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
     if (next != null) {
       throw new Error(`Busy processing sync rules - cannot reprocess`);
     }
 
-    const active = await storage.getActiveSyncRules();
+    const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
     if (active == null) {
       throw new errors.JourneyError({
         status: 422,
@@ -156,15 +127,18 @@ export const reprocess = routeDefinition({
       });
     }
 
-    const new_rules = await storage.updateSyncRules({
+    const new_rules = await activeBucketStorage.updateSyncRules({
       content: active.sync_rules.content
     });
 
+    const baseConfig = await apiHandler.getSourceConfig();
+
     return internal_routes.ReprocessResponse.encode({
       connections: [
         {
-          tag: system.config.connection!.tag,
-          id: system.config.connection!.id,
+          // Previously the connection was asserted with `!`
+          tag: baseConfig!.tag!,
+          id: baseConfig!.id,
           slot_name: new_rules.slot_name
         }
       ]
@@ -178,14 +152,16 @@ export const validate = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ValidateRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
+    const {
+      context: { service_context }
+    } = payload;
     const content = payload.params.sync_rules;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const schemaData = await api.getConnectionsSchema(system);
+    const schemaData = await api.getConnectionsSchema(apiHandler);
     const schema = new StaticSchema(schemaData.connections);
 
-    const sync_rules: PersistedSyncRulesContent = {
+    const sync_rules: storage.PersistedSyncRulesContent = {
       // Dummy values
       id: 0,
       slot_name: '',
@@ -193,7 +169,10 @@ export const validate = routeDefinition({
       parsed() {
         return {
           ...this,
-          sync_rules: SqlSyncRules.fromYaml(content, { throwOnError: false, schema })
+          sync_rules: SqlSyncRules.fromYaml(content, {
+            ...apiHandler.getParseSyncRulesOptions(),
+            schema
+          })
         };
       },
       sync_rules_content: content,
@@ -202,19 +181,24 @@ export const validate = routeDefinition({
       }
     };
 
-    const connectionStatus = await api.getConnectionStatus(system);
-    if (connectionStatus == null) {
+    const connectionStatus = await apiHandler.getConnectionStatus();
+    if (!connectionStatus) {
       return internal_routes.ValidateResponse.encode({
         errors: [{ level: 'fatal', message: 'No connection configured' }],
         connections: []
       });
     }
 
-    const status = (await api.getSyncRulesStatus(sync_rules, system, {
-      include_content: false,
-      check_connection: connectionStatus?.connected,
-      live_status: false
-    }))!;
+    const status = (await api.getSyncRulesStatus(
+      service_context.storageEngine.activeBucketStorage,
+      apiHandler,
+      sync_rules,
+      {
+        include_content: false,
+        check_connection: connectionStatus.connected,
+        live_status: false
+      }
+    ))!;
 
     if (connectionStatus == null) {
       status.errors.push({ level: 'fatal', message: 'No connection configured' });
@@ -224,14 +208,4 @@ export const validate = routeDefinition({
   }
 });
 
-function mapColumnValue(value: SqliteValue) {
-  if (typeof value == 'bigint') {
-    return Number(value);
-  } else if (isJsonValue(value)) {
-    return value;
-  } else {
-    return null;
-  }
-}
-
-export const ADMIN_ROUTES = [demoCredentials, executeSql, diagnostics, getSchema, reprocess, validate];
+export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate];

package/src/routes/endpoints/checkpointing.ts

@@ -1,11 +1,13 @@
 import * as t from 'ts-codec';
-import { router, schema } from '@powersync/lib-services-framework';
+import { logger, router, schema } from '@powersync/lib-services-framework';
 
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
-const WriteCheckpointRequest = t.object({});
+const WriteCheckpointRequest = t.object({
+  client_id: t.string.optional()
+});
 
 export const writeCheckpoint = routeDefinition({
   path: '/write-checkpoint.json',
@@ -13,13 +15,33 @@ export const writeCheckpoint = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-    const storage = system.storage;
+    const {
+      context: { service_context }
+    } = payload;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
-    return {
-      checkpoint
-    };
+    // This old API needs a persisted checkpoint id.
+    // Since we don't use LSNs anymore, the only way to get that is to wait.
+    const start = Date.now();
+
+    const head = await apiHandler.getReplicationHead();
+
+    const timeout = 50_000;
+
+    logger.info(`Waiting for LSN checkpoint: ${head}`);
+    while (Date.now() - start < timeout) {
+      const cp = await service_context.storageEngine.activeBucketStorage.getActiveCheckpoint();
+      if (!cp.hasSyncRules()) {
+        throw new Error('No sync rules available');
+      }
+      if (cp.lsn && cp.lsn >= head) {
+        logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);
+        return { checkpoint: cp.checkpoint };
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 30));
+    }
+    throw new Error('Timeout while waiting for checkpoint');
   }
 });
 
@@ -29,11 +51,23 @@ export const writeCheckpoint2 = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const { user_id, system } = payload.context;
-    const storage = system.storage;
-    const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, user_id!);
+    const { user_id, service_context } = payload.context;
+
+    const apiHandler = service_context.routerEngine!.getAPI();
+
+    const client_id = payload.params.client_id;
+    const full_user_id = util.checkpointUserId(user_id, client_id);
+
+    const currentCheckpoint = await apiHandler.getReplicationHead();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+
+    const writeCheckpoint = await activeBucketStorage.createWriteCheckpoint(full_user_id, { '1': currentCheckpoint });
+    logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`);
+
     return {
-      write_checkpoint: String(write_checkpoint)
+      write_checkpoint: String(writeCheckpoint)
     };
   }
 });

package/src/routes/endpoints/route-endpoints-index.ts

@@ -1,6 +1,5 @@
 export * from './admin.js';
 export * from './checkpointing.js';
-export * from './dev.js';
 export * from './socket-route.js';
 export * from './sync-rules.js';
 export * from './sync-stream.js';

package/src/routes/endpoints/socket-route.ts

@@ -11,10 +11,28 @@ import { SyncRoutes } from './sync-stream.js';
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
-    handler: async ({ context, params, responder, observer, initialN }) => {
-      const { system } = context;
+    handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
+      const { service_context } = context;
+      const { routerEngine } = service_context;
 
-      if (system.closed) {
+      // Create our own controller that we can abort directly
+      const controller = new AbortController();
+      upstreamSignal.addEventListener('abort', () => {
+        controller.abort();
+      });
+      if (upstreamSignal.aborted) {
+        controller.abort();
+      }
+      const signal = controller.signal;
+
+      let requestedN = initialN;
+      const disposer = observer.registerListener({
+        request(n) {
+          requestedN += n;
+        }
+      });
+
+      if (routerEngine!.closed) {
         responder.onError(
           new errors.JourneyError({
             status: 503,
@@ -26,13 +44,13 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
-      const controller = new AbortController();
-
       const syncParams = new RequestParameters(context.token_payload!, params.parameters ?? {});
 
-      const storage = system.storage;
+      const {
+        storageEngine: { activeBucketStorage }
+      } = service_context;
       // Sanity check before we start the stream
-      const cp = await storage.getActiveCheckpoint();
+      const cp = await activeBucketStorage.getActiveCheckpoint();
       if (!cp.hasSyncRules()) {
         responder.onError(
           new errors.JourneyError({
@@ -45,25 +63,16 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
-      let requestedN = initialN;
-      const disposer = observer.registerListener({
-        request(n) {
-          requestedN += n;
-        },
-        cancel: () => {
-          controller.abort();
-        }
-      });
-
-      const removeStopHandler = system.addStopHandler(() => {
-        observer.triggerCancel();
+      const removeStopHandler = routerEngine!.addStopHandler(() => {
+        controller.abort();
       });
 
       Metrics.getInstance().concurrent_connections.add(1);
       const tracker = new sync.RequestTracker();
       try {
         for await (const data of sync.streamResponse({
-          storage,
+          storage: activeBucketStorage,
+          parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
           params: {
             ...params,
             binary_data: true // always true for web sockets
@@ -75,8 +84,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             keep_alive: false
           },
           tracker,
-          signal: controller.signal
+          signal
         })) {
+          if (signal.aborted) {
+            break;
+          }
           if (data == null) {
             // Empty value just to flush iterator memory
             continue;
@@ -93,7 +105,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             tracker.addDataSynced(serialized.length);
           }
 
-          if (requestedN <= 0) {
+          if (requestedN <= 0 && !signal.aborted) {
             await new Promise<void>((resolve) => {
               const l = observer.registerListener({
                 request() {
@@ -101,14 +113,17 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
                     // Management of updating the total requested items is done above
                     resolve();
                     l();
+                    signal.removeEventListener('abort', onAbort);
                   }
-                },
-                cancel: () => {
-                  // Don't wait here if the request is cancelled
-                  resolve();
-                  l();
                 }
               });
+              const onAbort = () => {
+                // Don't wait here if the request is cancelled
+                resolve();
+                l();
+                signal.removeEventListener('abort', onAbort);
+              };
+              signal.addEventListener('abort', onAbort);
             });
           }
         }
@@ -124,6 +139,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         disposer();
         logger.info(`Sync stream complete`, {
           user_id: syncParams.user_id,
+          client_id: params.client_id,
+          user_agent: context.user_agent,
           operations_synced: tracker.operationsSynced,
           data_synced_bytes: tracker.dataSyncedBytes
         });