@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/src/routes/endpoints/socket-route.ts

@@ -1,27 +1,38 @@
-import { container, errors, logger, schema } from '@powersync/lib-services-framework';
+import { errors, logger, schema } from '@powersync/lib-services-framework';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { serialize } from 'bson';
 
 import { Metrics } from '../../metrics/Metrics.js';
-import { streamResponse } from '../../sync/sync.js';
+import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 import { SocketRouteGenerator } from '../router-socket.js';
 import { SyncRoutes } from './sync-stream.js';
-import { RequestTracker } from '../../sync/RequestTracker.js';
 
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
-    authorize: ({ context }) => {
-      return {
-        authorized: !!context.token_payload,
-        errors: ['Authentication required'].concat(context.token_errors ?? [])
-      };
-    },
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
-    handler: async ({ context, params, responder, observer, initialN }) => {
-      const { system } = context;
+    handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
+      const { service_context } = context;
+      const { routerEngine } = service_context;
 
-      if (system.closed) {
+      // Create our own controller that we can abort directly
+      const controller = new AbortController();
+      upstreamSignal.addEventListener('abort', () => {
+        controller.abort();
+      });
+      if (upstreamSignal.aborted) {
+        controller.abort();
+      }
+      const signal = controller.signal;
+
+      let requestedN = initialN;
+      const disposer = observer.registerListener({
+        request(n) {
+          requestedN += n;
+        }
+      });
+
+      if (routerEngine!.closed) {
         responder.onError(
           new errors.JourneyError({
             status: 503,
@@ -33,13 +44,13 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
-      const controller = new AbortController();
-
       const syncParams = new RequestParameters(context.token_payload!, params.parameters ?? {});
 
-      const storage = system.storage;
+      const {
+        storageEngine: { activeBucketStorage }
+      } = service_context;
       // Sanity check before we start the stream
-      const cp = await storage.getActiveCheckpoint();
+      const cp = await activeBucketStorage.getActiveCheckpoint();
       if (!cp.hasSyncRules()) {
         responder.onError(
           new errors.JourneyError({
@@ -52,27 +63,16 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
-      let requestedN = initialN;
-      const disposer = observer.registerListener({
-        request(n) {
-          requestedN += n;
-        },
-        cancel: () => {
-          controller.abort();
-        }
-      });
-
-      const removeStopHandler = system.addStopHandler(() => {
-        observer.triggerCancel();
+      const removeStopHandler = routerEngine!.addStopHandler(() => {
+        controller.abort();
       });
 
-      const metrics = container.getImplementation(Metrics);
-
-      metrics.concurrent_connections.add(1);
-      const tracker = new RequestTracker();
+      Metrics.getInstance().concurrent_connections.add(1);
+      const tracker = new sync.RequestTracker();
       try {
-        for await (const data of streamResponse({
-          storage,
+        for await (const data of sync.streamResponse({
+          storage: activeBucketStorage,
+          parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
           params: {
             ...params,
             binary_data: true // always true for web sockets
@@ -84,8 +84,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             keep_alive: false
           },
           tracker,
-          signal: controller.signal
+          signal
         })) {
+          if (signal.aborted) {
+            break;
+          }
           if (data == null) {
             // Empty value just to flush iterator memory
             continue;
@@ -102,7 +105,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             tracker.addDataSynced(serialized.length);
           }
 
-          if (requestedN <= 0) {
+          if (requestedN <= 0 && !signal.aborted) {
             await new Promise<void>((resolve) => {
               const l = observer.registerListener({
                 request() {
@@ -110,14 +113,17 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
                     // Management of updating the total requested items is done above
                     resolve();
                     l();
+                    signal.removeEventListener('abort', onAbort);
                   }
-                },
-                cancel: () => {
-                  // Don't wait here if the request is cancelled
-                  resolve();
-                  l();
                 }
               });
+              const onAbort = () => {
+                // Don't wait here if the request is cancelled
+                resolve();
+                l();
+                signal.removeEventListener('abort', onAbort);
+              };
+              signal.addEventListener('abort', onAbort);
             });
           }
         }
@@ -133,10 +139,12 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         disposer();
         logger.info(`Sync stream complete`, {
           user_id: syncParams.user_id,
+          client_id: params.client_id,
+          user_agent: context.user_agent,
           operations_synced: tracker.operationsSynced,
           data_synced_bytes: tracker.dataSyncedBytes
         });
-        metrics.concurrent_connections.add(-1);
+        Metrics.getInstance().concurrent_connections.add(-1);
       }
     }
   });

package/src/routes/endpoints/sync-rules.ts

@@ -1,12 +1,11 @@
-import * as t from 'ts-codec';
-import type { FastifyPluginAsync } from 'fastify';
-import * as pgwire from '@powersync/service-jpgwire';
 import { errors, router, schema } from '@powersync/lib-services-framework';
 import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
+import type { FastifyPluginAsync } from 'fastify';
+import * as t from 'ts-codec';
 
-import * as replication from '../../replication/replication-index.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
+import { RouteAPI } from '../../api/RouteAPI.js';
 
 const DeploySyncRulesRequest = t.object({
   content: t.string
@@ -39,7 +38,10 @@ export const deploySyncRules = routeDefinition({
   plugins: [yamlPlugin],
   validator: schema.createTsCodecValidator(DeploySyncRulesRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    if (payload.context.system.config.sync_rules.present) {
+    const { service_context } = payload.context;
+    const { storageEngine } = service_context;
+
+    if (service_context.configuration.sync_rules.present) {
       // If sync rules are configured via the config, disable deploy via the API.
       throw new errors.JourneyError({
         status: 422,
@@ -51,7 +53,12 @@ export const deploySyncRules = routeDefinition({
     const content = payload.params.content;
 
     try {
-      SqlSyncRules.fromYaml(payload.params.content);
+      const apiHandler = service_context.routerEngine!.getAPI();
+      SqlSyncRules.fromYaml(payload.params.content, {
+        ...apiHandler.getParseSyncRulesOptions(),
+        // We don't do any schema-level validation at this point
+        schema: undefined
+      });
     } catch (e) {
       throw new errors.JourneyError({
         status: 422,
@@ -61,7 +68,7 @@ export const deploySyncRules = routeDefinition({
       });
     }
 
-    const sync_rules = await payload.context.system.storage.updateSyncRules({
+    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({
       content: content
     });
 
@@ -84,8 +91,10 @@ export const validateSyncRules = routeDefinition({
   validator: schema.createTsCodecValidator(ValidateSyncRulesRequest, { allowAdditional: true }),
   handler: async (payload) => {
     const content = payload.params.content;
+    const { service_context } = payload.context;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const info = await debugSyncRules(payload.context.system.requirePgPool(), content);
+    const info = await debugSyncRules(apiHandler, content);
 
     return replyPrettyJson(info);
   }
@@ -96,8 +105,12 @@ export const currentSyncRules = routeDefinition({
   method: router.HTTPMethod.GET,
   authorize: authApi,
   handler: async (payload) => {
-    const storage = payload.context.system.storage;
-    const sync_rules = await storage.getActiveSyncRulesContent();
+    const { service_context } = payload.context;
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+
+    const sync_rules = await activeBucketStorage.getActiveSyncRulesContent();
     if (!sync_rules) {
       throw new errors.JourneyError({
         status: 422,
@@ -105,12 +118,12 @@ export const currentSyncRules = routeDefinition({
         description: 'No active sync rules'
       });
     }
-    const info = await debugSyncRules(payload.context.system.requirePgPool(), sync_rules.sync_rules_content);
-    const next = await storage.getNextSyncRulesContent();
 
-    const next_info = next
-      ? await debugSyncRules(payload.context.system.requirePgPool(), next.sync_rules_content)
-      : null;
+    const apiHandler = service_context.routerEngine!.getAPI();
+    const info = await debugSyncRules(apiHandler, sync_rules.sync_rules_content);
+    const next = await activeBucketStorage.getNextSyncRulesContent();
+
+    const next_info = next ? await debugSyncRules(apiHandler, next.sync_rules_content) : null;
 
     const response = {
       current: {
@@ -140,8 +153,11 @@ export const reprocessSyncRules = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(ReprocessSyncRulesRequest),
   handler: async (payload) => {
-    const storage = payload.context.system.storage;
-    const sync_rules = await storage.getActiveSyncRules();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = payload.context.service_context;
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
+    const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
     if (sync_rules == null) {
       throw new errors.JourneyError({
         status: 422,
@@ -150,7 +166,7 @@ export const reprocessSyncRules = routeDefinition({
       });
     }
 
-    const new_rules = await storage.updateSyncRules({
+    const new_rules = await activeBucketStorage.updateSyncRules({
       content: sync_rules.sync_rules.content
     });
     return {
@@ -169,15 +185,15 @@ function replyPrettyJson(payload: any) {
   });
 }
 
-async function debugSyncRules(db: pgwire.PgClient, sync_rules: string) {
+async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
   try {
-    const rules = SqlSyncRules.fromYaml(sync_rules);
-    const source_table_patterns = rules.getSourceTables();
-    const wc = new replication.WalConnection({
-      db: db,
-      sync_rules: rules
+    const rules = SqlSyncRules.fromYaml(sync_rules, {
+      ...apiHandler.getParseSyncRulesOptions(),
+      // No schema-based validation at this point
+      schema: undefined
     });
-    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+    const source_table_patterns = rules.getSourceTables();
+    const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
 
     return {
       valid: true,

package/src/routes/endpoints/sync-stream.ts

@@ -1,4 +1,4 @@
-import { container, errors, logger, router, schema } from '@powersync/lib-services-framework';
+import { errors, logger, router, schema } from '@powersync/lib-services-framework';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { Readable } from 'stream';
 
@@ -8,7 +8,6 @@ import * as util from '../../util/util-index.js';
 import { Metrics } from '../../metrics/Metrics.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
-import { RequestTracker } from '../../sync/RequestTracker.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -20,9 +19,13 @@ export const syncStreamed = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
+    const { service_context } = payload.context;
+    const { routerEngine, storageEngine } = service_context;
+    const headers = payload.request.headers;
+    const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
+    const clientId = payload.params.client_id;
 
-    if (system.closed) {
+    if (routerEngine!.closed) {
       throw new errors.JourneyError({
         status: 503,
         code: 'SERVICE_UNAVAILABLE',
@@ -33,9 +36,8 @@ export const syncStreamed = routeDefinition({
     const params: util.StreamingSyncRequest = payload.params;
     const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
 
-    const storage = system.storage;
     // Sanity check before we start the stream
-    const cp = await storage.getActiveCheckpoint();
+    const cp = await storageEngine.activeBucketStorage.getActiveCheckpoint();
     if (!cp.hasSyncRules()) {
       throw new errors.JourneyError({
         status: 500,
@@ -43,16 +45,16 @@ export const syncStreamed = routeDefinition({
         description: 'No sync rules available'
       });
     }
-    const metrics = container.getImplementation(Metrics);
     const controller = new AbortController();
-    const tracker = new RequestTracker();
+    const tracker = new sync.RequestTracker();
     try {
-      metrics.concurrent_connections.add(1);
+      Metrics.getInstance().concurrent_connections.add(1);
       const stream = Readable.from(
         sync.transformToBytesTracked(
           sync.ndjson(
             sync.streamResponse({
-              storage,
+              storage: storageEngine.activeBucketStorage,
+              parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
               params,
               syncParams,
               token: payload.context.token_payload!,
@@ -65,7 +67,7 @@ export const syncStreamed = routeDefinition({
         { objectMode: false, highWaterMark: 16 * 1024 }
       );
 
-      const deregister = system.addStopHandler(() => {
+      const deregister = routerEngine!.addStopHandler(() => {
         // This error is not currently propagated to the client
         controller.abort();
         stream.destroy(new Error('Shutting down system'));
@@ -90,9 +92,11 @@ export const syncStreamed = routeDefinition({
         data: stream,
         afterSend: async () => {
           controller.abort();
-          metrics.concurrent_connections.add(-1);
+          Metrics.getInstance().concurrent_connections.add(-1);
           logger.info(`Sync stream complete`, {
             user_id: syncParams.user_id,
+            client_id: clientId,
+            user_agent: userAgent,
             operations_synced: tracker.operationsSynced,
             data_synced_bytes: tracker.dataSyncedBytes
           });
@@ -100,7 +104,7 @@ export const syncStreamed = routeDefinition({
       });
     } catch (ex) {
       controller.abort();
-      metrics.concurrent_connections.add(-1);
+      Metrics.getInstance().concurrent_connections.add(-1);
     }
   }
 });

package/src/routes/route-register.ts

@@ -1,6 +1,6 @@
-import fastify from 'fastify';
+import type fastify from 'fastify';
 
-import { errors, router, HTTPMethod, logger } from '@powersync/lib-services-framework';
+import { errors, HTTPMethod, logger, router } from '@powersync/lib-services-framework';
 import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
 export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {

package/src/routes/router-socket.ts

@@ -1,13 +1,14 @@
+import { IReactiveStream, ReactiveSocketRouter } from '@powersync/service-rsocket-router';
 import * as t from 'ts-codec';
-import { ReactiveSocketRouter, IReactiveStream } from '@powersync/service-rsocket-router';
 
 import { Context } from './router.js';
 
-export const RSocketContextMeta = t.object({
-  token: t.string
-});
-
 /**
  * Creates a socket route handler given a router instance
  */
 export type SocketRouteGenerator = (router: ReactiveSocketRouter<Context>) => IReactiveStream;
+
+export const RSocketContextMeta = t.object({
+  token: t.string,
+  user_agent: t.string.optional()
+});

package/src/routes/router.ts

@@ -1,16 +1,20 @@
 import { router } from '@powersync/lib-services-framework';
 import * as auth from '../auth/auth-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { ServiceContext } from '../system/ServiceContext.js';
 
 /**
  * Common context for routes
  */
 export type Context = {
   user_id?: string;
-  system: CorePowerSyncSystem;
+  service_context: ServiceContext;
 
   token_payload?: auth.JwtPayload;
   token_errors?: string[];
+  /**
+   * Only on websocket endpoints.
+   */
+  user_agent?: string;
 };
 
 export type BasicRouterRequest = {
@@ -36,6 +40,7 @@ export type RequestEndpointHandlerPayload<
   request: Request;
 };
 
+export type RouteDefinition<I = any, O = any> = RequestEndpoint<I, O>;
 /**
  * Helper function for making generics work well when defining routes
  */

package/src/routes/routes-index.ts

@@ -1,6 +1,9 @@
 export * as auth from './auth.js';
+export * from './configure-fastify.js';
+export * from './configure-rsocket.js';
 export * as endpoints from './endpoints/route-endpoints-index.js';
 export * as hooks from './hooks.js';
 export * from './route-register.js';
 export * from './router-socket.js';
 export * from './router.js';
+export * from './RouterEngine.js';

package/src/runner/teardown.ts

@@ -1,108 +1,70 @@
 // Script to tear down the data when deleting an instance.
-// This deletes:
-// 1. The replication slots on the source postgres instance (if available).
-// 2. The mongo database.
+// This should:
+// 1. Attempt to clean up any remote configuration of data sources that was set up.
+// 2. Delete the storage
 
-import * as timers from 'timers/promises';
-
-import * as db from '../db/db-index.js';
+import { container, logger } from '@powersync/lib-services-framework';
+import * as modules from '../modules/modules-index.js';
+import * as system from '../system/system-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as utils from '../util/util-index.js';
-import * as replication from '../replication/replication-index.js';
-import { logger } from '@powersync/lib-services-framework';
+import timers from 'timers/promises';
 
-/**
- * Attempt to terminate a single sync rules instance.
- *
- * This may fail with a lock error.
- */
-async function terminateReplicator(
-  storageFactory: storage.BucketStorageFactory,
-  connection: utils.ResolvedConnection,
-  syncRules: storage.PersistedSyncRulesContent
-) {
-  // The lock may still be active if the current replication instance
-  // hasn't stopped yet.
-  const lock = await syncRules.lock();
+export async function teardown(runnerConfig: utils.RunnerConfig) {
   try {
-    const parsed = syncRules.parsed();
-    const storage = storageFactory.getInstance(parsed);
-    const stream = new replication.WalStreamRunner({
-      factory: storageFactory,
-      storage: storage,
-      source_db: connection,
-      lock
-    });
+    logger.info(`Tearing down PowerSync instance...`);
+    const config = await utils.loadConfig(runnerConfig);
+    const serviceContext = new system.ServiceContextContainer(config);
+    const moduleManager = container.getImplementation(modules.ModuleManager);
+    await moduleManager.initialize(serviceContext);
+    // This is mostly done to ensure that the storage is ready
+    await serviceContext.lifeCycleEngine.start();
 
-    logger.info(`Terminating replication slot ${stream.slot_name}`);
-    await stream.terminate();
-    logger.info(`Terminated replication slot ${stream.slot_name}`);
-  } finally {
-    await lock.release();
+    await terminateSyncRules(serviceContext.storageEngine.activeBucketStorage, moduleManager);
+    await serviceContext.storageEngine.activeStorage.tearDown();
+    logger.info(`Teardown complete.`);
+    process.exit(0);
+  } catch (e) {
+    logger.error(`Teardown failure`, e);
+    process.exit(1);
   }
 }
 
-/**
- * Terminate all replicating sync rules, deleting the replication slots.
- *
- * Retries lock and other errors for up to two minutes.
- *
- * This is a best-effot attempt. In some cases it may not be possible to delete the replication
- * slot, such as when the postgres instance is unreachable.
- */
-async function terminateReplicators(
-  storageFactory: storage.BucketStorageFactory,
-  connection: utils.ResolvedConnection
-) {
+async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, moduleManager: modules.ModuleManager) {
+  logger.info(`Terminating sync rules...`);
   const start = Date.now();
-  while (Date.now() - start < 12_000) {
+  const locks: storage.ReplicationLock[] = [];
+  while (Date.now() - start < 120_000) {
     let retry = false;
-    const replicationRules = await storageFactory.getReplicatingSyncRules();
-    for (let syncRules of replicationRules) {
-      try {
-        await terminateReplicator(storageFactory, connection, syncRules);
-      } catch (e) {
-        retry = true;
-        console.error(e);
-        logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
+    const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
+    // Lock all the replicating sync rules
+    for (const replicatingSyncRule of replicatingSyncRules) {
+      const lock = await replicatingSyncRule.lock();
+      locks.push(lock);
+    }
+
+    const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
+    const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
+    try {
+      // Clean up any module specific configuration for the sync rules
+      await moduleManager.tearDown({ syncRules: combinedSyncRules });
+
+      // Mark the sync rules as terminated
+      for (let syncRules of combinedSyncRules) {
+        const syncRulesStorage = storageFactory.getInstance(syncRules);
+        // The storage will be dropped at the end of the teardown, so we don't need to clear it here
+        await syncRulesStorage.terminate({ clearStorage: false });
+      }
+    } catch (e) {
+      retry = true;
+      for (const lock of locks) {
+        await lock.release();
       }
     }
+
     if (!retry) {
       break;
     }
     await timers.setTimeout(5_000);
   }
 }
-
-export async function teardown(runnerConfig: utils.RunnerConfig) {
-  const config = await utils.loadConfig(runnerConfig);
-  const mongoDB = storage.createPowerSyncMongo(config.storage);
-  try {
-    logger.info(`Waiting for auth`);
-    await db.mongo.waitForAuth(mongoDB.db);
-
-    const bucketStorage = new storage.MongoBucketStorage(mongoDB, { slot_name_prefix: config.slot_name_prefix });
-    const connection = config.connection;
-
-    logger.info(`Terminating replication slots`);
-
-    if (connection) {
-      await terminateReplicators(bucketStorage, connection);
-    }
-
-    const database = mongoDB.db;
-    logger.info(`Dropping database ${database.namespace}`);
-    await database.dropDatabase();
-    logger.info(`Done`);
-    await mongoDB.client.close();
-
-    // If there was an error connecting to postgress, the process may stay open indefinitely.
-    // This forces an exit.
-    // We do not consider those errors a teardown failure.
-    process.exit(0);
-  } catch (e) {
-    logger.error(`Teardown failure`, e);
-    await mongoDB.client.close();
-    process.exit(1);
-  }
-}