@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240725112650

package/src/replication/WalStream.ts

@@ -406,7 +406,7 @@ WHERE  oid = $1::regclass`,
         await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record });
       }
       at += rows.length;
-      container.getImplementation(Metrics).rows_replicated_total.add(rows.length);
+      Metrics.getInstance().rows_replicated_total.add(rows.length);
 
       await touch();
     }
@@ -492,21 +492,19 @@ WHERE  oid = $1::regclass`,
         return null;
       }
 
-      const metrics = container.getImplementation(Metrics);
-
       if (msg.tag == 'insert') {
-        metrics.rows_replicated_total.add(1);
+        Metrics.getInstance().rows_replicated_total.add(1);
         const baseRecord = util.constructAfterRecord(msg);
         return await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: baseRecord });
       } else if (msg.tag == 'update') {
-        metrics.rows_replicated_total.add(1);
+        Metrics.getInstance().rows_replicated_total.add(1);
         // "before" may be null if the replica id columns are unchanged
         // It's fine to treat that the same as an insert.
         const before = util.constructBeforeRecord(msg);
         const after = util.constructAfterRecord(msg);
         return await batch.save({ tag: 'update', sourceTable: table, before: before, after: after });
       } else if (msg.tag == 'delete') {
-        metrics.rows_replicated_total.add(1);
+        Metrics.getInstance().rows_replicated_total.add(1);
         const before = util.constructBeforeRecord(msg)!;
 
         return await batch.save({ tag: 'delete', sourceTable: table, before: before, after: undefined });
@@ -557,8 +555,6 @@ WHERE  oid = $1::regclass`,
     // Auto-activate as soon as initial replication is done
     await this.storage.autoActivate();
 
-    const metrics = container.getImplementation(Metrics);
-
     await this.storage.startBatch({}, async (batch) => {
       // Replication never starts in the middle of a transaction
       let inTx = false;
@@ -581,7 +577,7 @@ WHERE  oid = $1::regclass`,
           } else if (msg.tag == 'begin') {
             inTx = true;
           } else if (msg.tag == 'commit') {
-            metrics.transactions_replicated_total.add(1);
+            Metrics.getInstance().transactions_replicated_total.add(1);
             inTx = false;
             await batch.commit(msg.lsn!);
             await this.ack(msg.lsn!, replicationStream);
@@ -606,7 +602,7 @@ WHERE  oid = $1::regclass`,
           }
         }
 
-        metrics.chunks_replicated_total.add(1);
+        Metrics.getInstance().chunks_replicated_total.add(1);
       }
     });
   }

package/src/routes/configure-fastify.ts

@@ -0,0 +1,102 @@
+import type fastify from 'fastify';
+import { registerFastifyRoutes } from './route-register.js';
+
+import * as system from '../system/system-index.js';
+
+import { ADMIN_ROUTES } from './endpoints/admin.js';
+import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
+import { DEV_ROUTES } from './endpoints/dev.js';
+import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
+import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
+import { createRequestQueueHook, CreateRequestQueueParams } from './hooks.js';
+import { RouteDefinition } from './router.js';
+
+/**
+ * A list of route definitions to be registered as endpoints.
+ * Supplied concurrency limits will be applied to the grouped routes.
+ */
+export type RouteRegistrationOptions = {
+  routes: RouteDefinition[];
+  queueOptions: CreateRequestQueueParams;
+};
+
+/**
+ * HTTP routes separated by API and Sync stream categories.
+ * This allows for separate concurrency limits.
+ */
+export type RouteDefinitions = {
+  api?: Partial<RouteRegistrationOptions>;
+  syncStream?: Partial<RouteRegistrationOptions>;
+};
+
+export type FastifyServerConfig = {
+  system: system.CorePowerSyncSystem;
+  routes?: RouteDefinitions;
+};
+
+export const DEFAULT_ROUTE_OPTIONS = {
+  api: {
+    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...DEV_ROUTES, ...SYNC_RULES_ROUTES],
+    queueOptions: {
+      concurrency: 10,
+      max_queue_depth: 20
+    }
+  },
+  syncStream: {
+    routes: [...SYNC_STREAM_ROUTES],
+    queueOptions: {
+      concurrency: 200,
+      max_queue_depth: 0
+    }
+  }
+};
+
+/**
+ * Registers default routes on a Fastify server. Consumers can optionally configure
+ * concurrency queue limits or override routes.
+ */
+export function configureFastifyServer(server: fastify.FastifyInstance, options: FastifyServerConfig) {
+  const { system, routes = DEFAULT_ROUTE_OPTIONS } = options;
+  /**
+   * Fastify creates an encapsulated context for each `.register` call.
+   * Creating a separate context here to separate the concurrency limits for Admin APIs
+   * and Sync Streaming routes.
+   * https://github.com/fastify/fastify/blob/main/docs/Reference/Encapsulation.md
+   */
+  server.register(async function (childContext) {
+    registerFastifyRoutes(
+      childContext,
+      async () => {
+        return {
+          user_id: undefined,
+          system: system
+        };
+      },
+      routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes
+    );
+    // Limit the active concurrent requests
+    childContext.addHook(
+      'onRequest',
+      createRequestQueueHook(routes.api?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.api.queueOptions)
+    );
+  });
+
+  // Create a separate context for concurrency queueing
+  server.register(async function (childContext) {
+    registerFastifyRoutes(
+      childContext,
+      async () => {
+        return {
+          user_id: undefined,
+          system: system
+        };
+      },
+      routes.syncStream?.routes ?? DEFAULT_ROUTE_OPTIONS.syncStream.routes
+    );
+    // Limit the active concurrent requests
+    childContext.addHook(
+      'onRequest',
+      createRequestQueueHook(routes.syncStream?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.syncStream.queueOptions)
+    );
+  });
+}

package/src/routes/configure-rsocket.ts

@@ -0,0 +1,59 @@
+import { deserialize } from 'bson';
+import * as http from 'http';
+
+import { errors, logger } from '@powersync/lib-services-framework';
+import { ReactiveSocketRouter, RSocketRequestMeta } from '@powersync/service-rsocket-router';
+
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { generateContext, getTokenFromHeader } from './auth.js';
+import { syncStreamReactive } from './endpoints/socket-route.js';
+import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
+import { Context } from './router.js';
+
+export type RSockerRouterConfig = {
+  system: CorePowerSyncSystem;
+  server: http.Server;
+  routeGenerators?: SocketRouteGenerator[];
+};
+
+export const DEFAULT_SOCKET_ROUTES = [syncStreamReactive];
+
+export function configureRSocket(router: ReactiveSocketRouter<Context>, options: RSockerRouterConfig) {
+  const { routeGenerators = DEFAULT_SOCKET_ROUTES, server, system } = options;
+
+  router.applyWebSocketEndpoints(server, {
+    contextProvider: async (data: Buffer) => {
+      const { token } = RSocketContextMeta.decode(deserialize(data) as any);
+
+      if (!token) {
+        throw new errors.AuthorizationError('No token provided');
+      }
+
+      try {
+        const extracted_token = getTokenFromHeader(token);
+        if (extracted_token != null) {
+          const { context, errors: token_errors } = await generateContext(system, extracted_token);
+          if (context?.token_payload == null) {
+            throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
+          }
+          return {
+            token,
+            ...context,
+            token_errors: token_errors,
+            system
+          };
+        } else {
+          throw new errors.AuthorizationError('No token provided');
+        }
+      } catch (ex) {
+        logger.error(ex);
+        throw ex;
+      }
+    },
+    endpoints: routeGenerators.map((generator) => generator(router)),
+    metaDecoder: async (meta: Buffer) => {
+      return RSocketRequestMeta.decode(deserialize(meta) as any);
+    },
+    payloadDecoder: async (rawData?: Buffer) => rawData && deserialize(rawData)
+  });
+}

package/src/routes/endpoints/socket-route.ts

@@ -1,22 +1,15 @@
-import { container, errors, logger, schema } from '@powersync/lib-services-framework';
+import { errors, logger, schema } from '@powersync/lib-services-framework';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { serialize } from 'bson';
 
 import { Metrics } from '../../metrics/Metrics.js';
-import { streamResponse } from '../../sync/sync.js';
+import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 import { SocketRouteGenerator } from '../router-socket.js';
 import { SyncRoutes } from './sync-stream.js';
-import { RequestTracker } from '../../sync/RequestTracker.js';
 
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
-    authorize: ({ context }) => {
-      return {
-        authorized: !!context.token_payload,
-        errors: ['Authentication required'].concat(context.token_errors ?? [])
-      };
-    },
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
     handler: async ({ context, params, responder, observer, initialN }) => {
       const { system } = context;
@@ -66,12 +59,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         observer.triggerCancel();
       });
 
-      const metrics = container.getImplementation(Metrics);
-
-      metrics.concurrent_connections.add(1);
-      const tracker = new RequestTracker();
+      Metrics.getInstance().concurrent_connections.add(1);
+      const tracker = new sync.RequestTracker();
       try {
-        for await (const data of streamResponse({
+        for await (const data of sync.streamResponse({
           storage,
           params: {
             ...params,
@@ -136,7 +127,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
           operations_synced: tracker.operationsSynced,
           data_synced_bytes: tracker.dataSyncedBytes
         });
-        metrics.concurrent_connections.add(-1);
+        Metrics.getInstance().concurrent_connections.add(-1);
       }
     }
   });

package/src/routes/endpoints/sync-stream.ts

@@ -1,4 +1,4 @@
-import { container, errors, logger, router, schema } from '@powersync/lib-services-framework';
+import { errors, logger, router, schema } from '@powersync/lib-services-framework';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { Readable } from 'stream';
 
@@ -43,11 +43,10 @@ export const syncStreamed = routeDefinition({
         description: 'No sync rules available'
       });
     }
-    const metrics = container.getImplementation(Metrics);
     const controller = new AbortController();
     const tracker = new RequestTracker();
     try {
-      metrics.concurrent_connections.add(1);
+      Metrics.getInstance().concurrent_connections.add(1);
       const stream = Readable.from(
         sync.transformToBytesTracked(
           sync.ndjson(
@@ -90,7 +89,7 @@ export const syncStreamed = routeDefinition({
         data: stream,
         afterSend: async () => {
           controller.abort();
-          metrics.concurrent_connections.add(-1);
+          Metrics.getInstance().concurrent_connections.add(-1);
           logger.info(`Sync stream complete`, {
             user_id: syncParams.user_id,
             operations_synced: tracker.operationsSynced,
@@ -100,7 +99,7 @@ export const syncStreamed = routeDefinition({
       });
     } catch (ex) {
       controller.abort();
-      metrics.concurrent_connections.add(-1);
+      Metrics.getInstance().concurrent_connections.add(-1);
     }
   }
 });

package/src/routes/route-register.ts

@@ -1,6 +1,6 @@
-import fastify from 'fastify';
+import type fastify from 'fastify';
 
-import { errors, router, HTTPMethod, logger } from '@powersync/lib-services-framework';
+import { errors, HTTPMethod, logger, router } from '@powersync/lib-services-framework';
 import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
 export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {

package/src/routes/router-socket.ts

@@ -1,13 +1,13 @@
+import { IReactiveStream, ReactiveSocketRouter } from '@powersync/service-rsocket-router';
 import * as t from 'ts-codec';
-import { ReactiveSocketRouter, IReactiveStream } from '@powersync/service-rsocket-router';
 
 import { Context } from './router.js';
 
-export const RSocketContextMeta = t.object({
-  token: t.string
-});
-
 /**
  * Creates a socket route handler given a router instance
  */
 export type SocketRouteGenerator = (router: ReactiveSocketRouter<Context>) => IReactiveStream;
+
+export const RSocketContextMeta = t.object({
+  token: t.string
+});

package/src/routes/router.ts

@@ -36,6 +36,8 @@ export type RequestEndpointHandlerPayload<
   request: Request;
 };
 
+export type RouteDefinition<I = any, O = any> = RequestEndpoint<I, O>;
+
 /**
  * Helper function for making generics work well when defining routes
  */

package/src/routes/routes-index.ts

@@ -1,4 +1,6 @@
 export * as auth from './auth.js';
+export * from './configure-fastify.js';
+export * from './configure-rsocket.js';
 export * as endpoints from './endpoints/route-endpoints-index.js';
 export * as hooks from './hooks.js';
 export * from './route-register.js';

package/src/storage/BucketStorage.ts

@@ -228,7 +228,7 @@ export interface SyncRulesBucketStorage {
     checkpoint: util.OpId,
     dataBuckets: Map<string, string>,
     options?: BucketDataBatchOptions
-  ): AsyncIterable<util.SyncBucketData>;
+  ): AsyncIterable<SyncBucketDataBatch>;
 
   /**
    * Compute checksums for a given list of buckets.
@@ -266,6 +266,8 @@ export interface SyncRulesBucketStorage {
    * Errors are cleared on commit.
    */
   reportError(e: any): Promise<void>;
+
+  compact(options?: CompactOptions): Promise<void>;
 }
 
 export interface SyncRuleStatus {
@@ -388,6 +390,11 @@ export interface SaveDelete {
   after?: undefined;
 }
 
+export interface SyncBucketDataBatch {
+  batch: util.SyncBucketData;
+  targetOp: bigint | null;
+}
+
 export function mergeToast(record: ToastableSqliteRow, persisted: ToastableSqliteRow): ToastableSqliteRow {
   const newRecord: ToastableSqliteRow = {};
   for (let key in record) {
@@ -399,3 +406,31 @@ export function mergeToast(record: ToastableSqliteRow, persisted: ToastableSqlit
   }
   return newRecord;
 }
+
+export interface CompactOptions {
+  /**
+   * Heap memory limit for the compact process.
+   *
+   * Add around 64MB to this to determine the "--max-old-space-size" argument.
+   * Add another 80MB to get RSS usage / memory limits.
+   */
+  memoryLimitMB?: number;
+
+  /**
+   * If specified, ignore any operations newer than this when compacting.
+   *
+   * This is primarily for tests, where we want to test compacting at a specific
+   * point.
+   *
+   * This can also be used to create a "safe buffer" of recent operations that should
+   * not be compacted, to avoid invalidating checkpoints in use.
+   */
+  maxOpId?: bigint;
+
+  /**
+   * If specified, compact only the specific buckets.
+   *
+   * If not specified, compacts all buckets.
+   */
+  compactBuckets?: string[];
+}