@powersync/service-core 0.0.0-dev-20240620165206

package/src/routes/endpoints/sync-rules.ts

@@ -0,0 +1,227 @@
+import * as t from 'ts-codec';
+import type { FastifyPluginAsync } from 'fastify';
+import * as pgwire from '@powersync/service-jpgwire';
+import { errors, router, schema } from '@powersync/lib-services-framework';
+import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
+
+import * as replication from '../../replication/replication-index.js';
+import { authApi } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+const DeploySyncRulesRequest = t.object({
+  content: t.string
+});
+
+export const yamlPlugin: FastifyPluginAsync = async (fastify) => {
+  fastify.addContentTypeParser('application/yaml', async (request, payload, _d) => {
+    const data: any[] = [];
+    for await (const chunk of payload) {
+      data.push(chunk);
+    }
+
+    request.params = { content: Buffer.concat(data).toString('utf8') };
+  });
+};
+
+/**
+ * Declares the plugin should be available on the same scope
+ * without requiring the `fastify-plugin` package as a dependency.
+ * https://fastify.dev/docs/latest/Reference/Plugins/#handle-the-scope
+ */
+//@ts-expect-error
+yamlPlugin[Symbol.for('skip-override')] = true;
+
+export const deploySyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/deploy',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  parse: true,
+  plugins: [yamlPlugin],
+  validator: schema.createTsCodecValidator(DeploySyncRulesRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    if (payload.context.system.config.sync_rules.present) {
+      // If sync rules are configured via the config, disable deploy via the API.
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'API_DISABLED',
+        description: 'Sync rules API disabled',
+        details: 'Use the management API to deploy sync rules'
+      });
+    }
+    const content = payload.params.content;
+
+    try {
+      SqlSyncRules.fromYaml(payload.params.content);
+    } catch (e) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'INVALID_SYNC_RULES',
+        description: 'Sync rules parsing failed',
+        details: e.message
+      });
+    }
+
+    const sync_rules = await payload.context.system.storage.updateSyncRules({
+      content: content
+    });
+
+    return {
+      slot_name: sync_rules.slot_name
+    };
+  }
+});
+
+const ValidateSyncRulesRequest = t.object({
+  content: t.string
+});
+
+export const validateSyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/validate',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  parse: true,
+  plugins: [yamlPlugin],
+  validator: schema.createTsCodecValidator(ValidateSyncRulesRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const content = payload.params.content;
+
+    const info = await debugSyncRules(payload.context.system.requirePgPool(), content);
+
+    return replyPrettyJson(info);
+  }
+});
+
+export const currentSyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/current',
+  method: router.HTTPMethod.GET,
+  authorize: authApi,
+  handler: async (payload) => {
+    const storage = payload.context.system.storage;
+    const sync_rules = await storage.getActiveSyncRulesContent();
+    if (!sync_rules) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'NO_SYNC_RULES',
+        description: 'No active sync rules'
+      });
+    }
+    const info = await debugSyncRules(payload.context.system.requirePgPool(), sync_rules.sync_rules_content);
+    const next = await storage.getNextSyncRulesContent();
+
+    const next_info = next
+      ? await debugSyncRules(payload.context.system.requirePgPool(), next.sync_rules_content)
+      : null;
+
+    const response = {
+      current: {
+        slot_name: sync_rules.slot_name,
+        content: sync_rules.sync_rules_content,
+        ...info
+      },
+      next:
+        next == null
+          ? null
+          : {
+              slot_name: next.slot_name,
+              content: next.sync_rules_content,
+              ...next_info
+            }
+    };
+
+    return replyPrettyJson({ data: response });
+  }
+});
+
+const ReprocessSyncRulesRequest = t.object({});
+
+export const reprocessSyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/reprocess',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(ReprocessSyncRulesRequest),
+  handler: async (payload) => {
+    const storage = payload.context.system.storage;
+    const sync_rules = await storage.getActiveSyncRules();
+    if (sync_rules == null) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'NO_SYNC_RULES',
+        description: 'No active sync rules'
+      });
+    }
+
+    const new_rules = await storage.updateSyncRules({
+      content: sync_rules.sync_rules.content
+    });
+    return {
+      slot_name: new_rules.slot_name
+    };
+  }
+});
+
+export const SYNC_RULES_ROUTES = [validateSyncRules, deploySyncRules, reprocessSyncRules, currentSyncRules];
+
+function replyPrettyJson(payload: any) {
+  return new router.RouterResponse({
+    status: 200,
+    data: JSON.stringify(payload, null, 2) + '\n',
+    headers: { 'Content-Type': 'application/json' }
+  });
+}
+
+async function debugSyncRules(db: pgwire.PgClient, sync_rules: string) {
+  try {
+    const rules = SqlSyncRules.fromYaml(sync_rules);
+    const source_table_patterns = rules.getSourceTables();
+    const wc = new replication.WalConnection({
+      db: db,
+      sync_rules: rules
+    });
+    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+
+    return {
+      valid: true,
+      bucket_definitions: rules.bucket_descriptors.map((d) => {
+        let all_parameter_queries = [...d.parameter_queries.values()].flat();
+        let all_data_queries = [...d.data_queries.values()].flat();
+        return {
+          name: d.name,
+          bucket_parameters: d.bucket_parameters,
+          global_parameter_queries: d.global_parameter_queries.map((q) => {
+            return {
+              sql: q.sql
+            };
+          }),
+          parameter_queries: all_parameter_queries.map((q) => {
+            return {
+              sql: q.sql,
+              table: q.sourceTable,
+              input_parameters: q.input_parameters
+            };
+          }),
+
+          data_queries: all_data_queries.map((q) => {
+            return {
+              sql: q.sql,
+              table: q.sourceTable,
+              columns: q.columnOutputNames()
+            };
+          })
+        };
+      }),
+      source_tables: resolved_tables,
+      data_tables: rules.debugGetOutputTables()
+    };
+  } catch (e) {
+    if (e instanceof SyncRulesErrors) {
+      return {
+        valid: false,
+        errors: e.errors.map((e) => e.message)
+      };
+    }
+    return {
+      valid: false,
+      errors: [e.message]
+    };
+  }
+}

package/src/routes/endpoints/sync-stream.ts

@@ -0,0 +1,101 @@
+import { Readable } from 'stream';
+import { SyncParameters, normalizeTokenParameters } from '@powersync/service-sync-rules';
+import { errors, logger, router, schema } from '@powersync/lib-services-framework';
+
+import * as sync from '../../sync/sync-index.js';
+import * as util from '../../util/util-index.js';
+
+import { authUser } from '../auth.js';
+import { routeDefinition } from '../router.js';
+import { Metrics } from '../../metrics/Metrics.js';
+
+export enum SyncRoutes {
+  STREAM = '/sync/stream'
+}
+
+export const syncStreamed = routeDefinition({
+  path: SyncRoutes.STREAM,
+  method: router.HTTPMethod.POST,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    if (system.closed) {
+      throw new errors.JourneyError({
+        status: 503,
+        code: 'SERVICE_UNAVAILABLE',
+        description: 'Service temporarily unavailable'
+      });
+    }
+
+    const params: util.StreamingSyncRequest = payload.params;
+    const syncParams: SyncParameters = normalizeTokenParameters(
+      payload.context.token_payload!.parameters ?? {},
+      payload.params.parameters ?? {}
+    );
+
+    const storage = system.storage;
+    // Sanity check before we start the stream
+    const cp = await storage.getActiveCheckpoint();
+    if (!cp.hasSyncRules()) {
+      throw new errors.JourneyError({
+        status: 500,
+        code: 'NO_SYNC_RULES',
+        description: 'No sync rules available'
+      });
+    }
+    const controller = new AbortController();
+    try {
+      Metrics.getInstance().concurrent_connections.add(1);
+      const stream = Readable.from(
+        sync.transformToBytesTracked(
+          sync.ndjson(
+            sync.streamResponse({
+              storage,
+              params,
+              syncParams,
+              token: payload.context.token_payload!,
+              signal: controller.signal
+            })
+          )
+        ),
+        { objectMode: false, highWaterMark: 16 * 1024 }
+      );
+
+      const deregister = system.addStopHandler(() => {
+        // This error is not currently propagated to the client
+        controller.abort();
+        stream.destroy(new Error('Shutting down system'));
+      });
+      stream.on('close', () => {
+        deregister();
+      });
+
+      stream.on('error', (error) => {
+        controller.abort();
+        // Note: This appears as a 200 response in the logs.
+        if (error.message != 'Shutting down system') {
+          logger.error('Streaming sync request failed', error);
+        }
+      });
+
+      return new router.RouterResponse({
+        status: 200,
+        headers: {
+          'Content-Type': 'application/x-ndjson'
+        },
+        data: stream,
+        afterSend: async () => {
+          controller.abort();
+          Metrics.getInstance().concurrent_connections.add(-1);
+        }
+      });
+    } catch (ex) {
+      controller.abort();
+      Metrics.getInstance().concurrent_connections.add(-1);
+    }
+  }
+});
+
+export const SYNC_STREAM_ROUTES = [syncStreamed];

package/src/routes/hooks.ts

@@ -0,0 +1,46 @@
+import type fastify from 'fastify';
+import a from 'async';
+import { logger } from '@powersync/lib-services-framework';
+
+export type CreateRequestQueueParams = {
+  max_queue_depth: number;
+  concurrency: number;
+};
+
+/**
+ * Creates a request queue which limits the amount of concurrent connections which
+ * are active at any time.
+ */
+export const createRequestQueueHook = (params: CreateRequestQueueParams): fastify.onRequestHookHandler => {
+  const request_queue = a.queue<() => Promise<void>>((event, done) => {
+    event().finally(done);
+  }, params.concurrency);
+
+  return (request, reply, next) => {
+    if (
+      (params.max_queue_depth == 0 && request_queue.running() == params.concurrency) ||
+      (params.max_queue_depth > 0 && request_queue.length() >= params.max_queue_depth)
+    ) {
+      logger.warn(`${request.method} ${request.url}`, {
+        status: 429,
+        method: request.method,
+        path: request.url,
+        route: request.routerPath,
+        queue_overflow: true
+      });
+      return reply.status(429).send();
+    }
+
+    const finished = new Promise<void>((resolve) => {
+      reply.then(
+        () => resolve(),
+        () => resolve()
+      );
+    });
+
+    request_queue.push(() => {
+      next();
+      return finished;
+    });
+  };
+};

package/src/routes/route-register.ts

@@ -0,0 +1,104 @@
+import fastify from 'fastify';
+
+import { errors, router, HTTPMethod, logger } from '@powersync/lib-services-framework';
+import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
+
+export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {
+  parse?: boolean;
+  plugins?: fastify.FastifyPluginAsync[];
+};
+
+/**
+ * Registers endpoint definitions as routes on a Fastify app instance.
+ */
+export function registerFastifyRoutes(
+  app: fastify.FastifyInstance,
+  contextProvider: ContextProvider,
+  endpoints: FastifyEndpoint<any, any, Context>[]
+) {
+  for (const e of endpoints) {
+    // Create a new context for each route
+    app.register(async function (fastify) {
+      fastify.route({
+        url: e.path,
+        method: e.method as HTTPMethod,
+        handler: async (request, reply) => {
+          const startTime = new Date();
+          let response: router.RouterResponse;
+          try {
+            const context = await contextProvider(request);
+
+            let combined = {
+              ...(request.params as any),
+              ...(request.query as any)
+            };
+
+            if (typeof request.body === 'object' && !Buffer.isBuffer(request.body) && !Array.isArray(request.body)) {
+              combined = {
+                ...combined,
+                ...request.body
+              };
+            }
+
+            const payload: RequestEndpointHandlerPayload = {
+              context: context,
+              params: combined,
+              request
+            };
+
+            const endpointResponse = await router.executeEndpoint(e, payload);
+
+            if (router.RouterResponse.isRouterResponse(endpointResponse)) {
+              response = endpointResponse;
+            } else if (router.isAsyncIterable(endpointResponse) || Buffer.isBuffer(endpointResponse)) {
+              response = new router.RouterResponse({
+                status: 200,
+                data: endpointResponse
+              });
+            } else {
+              response = new router.RouterResponse({
+                status: 200,
+                data: { data: endpointResponse }
+              });
+            }
+          } catch (ex) {
+            const journeyError = errors.JourneyError.isJourneyError(ex) ? ex : new errors.InternalServerError(ex);
+
+            response = new router.RouterResponse({
+              status: journeyError.errorData.status || 500,
+              headers: {
+                'Content-Type': 'application/json'
+              },
+              data: {
+                error: journeyError.errorData
+              }
+            });
+          }
+
+          Object.keys(response.headers).forEach((key) => {
+            reply.header(key, response.headers[key]);
+          });
+          reply.status(response.status);
+          try {
+            await reply.send(response.data);
+          } finally {
+            await response.afterSend?.();
+            logger.info(`${e.method} ${request.url}`, {
+              duration_ms: Math.round(new Date().valueOf() - startTime.valueOf() + Number.EPSILON),
+              status: response.status,
+              method: e.method,
+              path: request.url,
+              route: e.path
+            });
+          }
+        }
+      });
+
+      if (!(e.parse ?? true)) {
+        fastify.removeAllContentTypeParsers();
+      }
+
+      e.plugins?.forEach((plugin) => fastify.register(plugin));
+    });
+  }
+}

package/src/routes/router-socket.ts

@@ -0,0 +1,13 @@
+import * as t from 'ts-codec';
+import { ReactiveSocketRouter, IReactiveStream } from '@powersync/service-rsocket-router';
+
+import { Context } from './router.js';
+
+export const RSocketContextMeta = t.object({
+  token: t.string
+});
+
+/**
+ * Creates a socket route handler given a router instance
+ */
+export type SocketRouteGenerator = (router: ReactiveSocketRouter<Context>) => IReactiveStream;

package/src/routes/router.ts

@@ -0,0 +1,46 @@
+import { router } from '@powersync/lib-services-framework';
+import * as auth from '../auth/auth-index.js';
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+
+/**
+ * Common context for routes
+ */
+export type Context = {
+  user_id?: string;
+  system: CorePowerSyncSystem;
+
+  token_payload?: auth.JwtPayload;
+  token_errors?: string[];
+};
+
+export type BasicRouterRequest = {
+  headers: Record<string, string | string[] | undefined>;
+  protocol: string;
+  hostname: string;
+};
+
+export type ContextProvider = (request: BasicRouterRequest) => Promise<Context>;
+
+export type RequestEndpoint<
+  I,
+  O,
+  C = Context,
+  Payload = RequestEndpointHandlerPayload<I, C, BasicRouterRequest>
+> = router.Endpoint<I, O, C, Payload> & {};
+
+export type RequestEndpointHandlerPayload<
+  I = any,
+  C = Context,
+  Request = BasicRouterRequest
+> = router.EndpointHandlerPayload<I, C> & {
+  request: Request;
+};
+
+/**
+ * Helper function for making generics work well when defining routes
+ */
+export function routeDefinition<I, O, C = Context, Extension = {}>(
+  params: RequestEndpoint<I, O, C> & Extension
+): RequestEndpoint<I, O, C> & Extension {
+  return params;
+}

package/src/routes/routes-index.ts

@@ -0,0 +1,6 @@
+export * as auth from './auth.js';
+export * as endpoints from './endpoints/route-endpoints-index.js';
+export * as hooks from './hooks.js';
+export * from './route-register.js';
+export * from './router-socket.js';
+export * from './router.js';

package/src/runner/teardown.ts

@@ -0,0 +1,108 @@
+// Script to tear down the data when deleting an instance.
+// This deletes:
+// 1. The replication slots on the source postgres instance (if available).
+// 2. The mongo database.
+
+import * as timers from 'timers/promises';
+
+import * as db from '../db/db-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as utils from '../util/util-index.js';
+import * as replication from '../replication/replication-index.js';
+import { logger } from '@powersync/lib-services-framework';
+
+/**
+ * Attempt to terminate a single sync rules instance.
+ *
+ * This may fail with a lock error.
+ */
+async function terminateReplicator(
+  storageFactory: storage.BucketStorageFactory,
+  connection: utils.ResolvedConnection,
+  syncRules: storage.PersistedSyncRulesContent
+) {
+  // The lock may still be active if the current replication instance
+  // hasn't stopped yet.
+  const lock = await syncRules.lock();
+  try {
+    const parsed = syncRules.parsed();
+    const storage = storageFactory.getInstance(parsed);
+    const stream = new replication.WalStreamRunner({
+      factory: storageFactory,
+      storage: storage,
+      source_db: connection,
+      lock
+    });
+
+    logger.info(`Terminating replication slot ${stream.slot_name}`);
+    await stream.terminate();
+    logger.info(`Terminated replication slot ${stream.slot_name}`);
+  } finally {
+    await lock.release();
+  }
+}
+
+/**
+ * Terminate all replicating sync rules, deleting the replication slots.
+ *
+ * Retries lock and other errors for up to two minutes.
+ *
+ * This is a best-effot attempt. In some cases it may not be possible to delete the replication
+ * slot, such as when the postgres instance is unreachable.
+ */
+async function terminateReplicators(
+  storageFactory: storage.BucketStorageFactory,
+  connection: utils.ResolvedConnection
+) {
+  const start = Date.now();
+  while (Date.now() - start < 12_000) {
+    let retry = false;
+    const replicationRules = await storageFactory.getReplicatingSyncRules();
+    for (let syncRules of replicationRules) {
+      try {
+        await terminateReplicator(storageFactory, connection, syncRules);
+      } catch (e) {
+        retry = true;
+        console.error(e);
+        logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
+      }
+    }
+    if (!retry) {
+      break;
+    }
+    await timers.setTimeout(5_000);
+  }
+}
+
+export async function teardown(runnerConfig: utils.RunnerConfig) {
+  const config = await utils.loadConfig(runnerConfig);
+  const mongoDB = storage.createPowerSyncMongo(config.storage);
+  try {
+    logger.info(`Waiting for auth`);
+    await db.mongo.waitForAuth(mongoDB.db);
+
+    const bucketStorage = new storage.MongoBucketStorage(mongoDB, { slot_name_prefix: config.slot_name_prefix });
+    const connection = config.connection;
+
+    logger.info(`Terminating replication slots`);
+
+    if (connection) {
+      await terminateReplicators(bucketStorage, connection);
+    }
+
+    const database = mongoDB.db;
+    logger.info(`Dropping database ${database.namespace}`);
+    await database.dropDatabase();
+    logger.info(`Done`);
+    await mongoDB.client.close();
+
+    // If there was an error connecting to postgress, the process may stay open indefinitely.
+    // This forces an exit.
+    // We do not consider those errors a teardown failure.
+    process.exit(0);
+  } catch (e) {
+    logger.error(`Teardown failure`, e);
+    await mongoDB.client.close();
+    process.exit(1);
+  }
+}