@powersync/service-core 0.2.2 → 0.4.0

package/src/routes/endpoints/checkpointing.ts

@@ -0,0 +1,41 @@
+import * as t from 'ts-codec';
+import { router, schema } from '@powersync/lib-services-framework';
+
+import * as util from '../../util/util-index.js';
+import { authUser } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+const WriteCheckpointRequest = t.object({});
+
+export const writeCheckpoint = routeDefinition({
+  path: '/write-checkpoint.json',
+  method: router.HTTPMethod.GET,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+    const storage = system.storage;
+
+    const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
+    return {
+      checkpoint
+    };
+  }
+});
+
+export const writeCheckpoint2 = routeDefinition({
+  path: '/write-checkpoint2.json',
+  method: router.HTTPMethod.GET,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const { user_id, system } = payload.context;
+    const storage = system.storage;
+    const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, user_id!);
+    return {
+      write_checkpoint: String(write_checkpoint)
+    };
+  }
+});
+
+export const CHECKPOINT_ROUTES = [writeCheckpoint, writeCheckpoint2];

package/src/routes/endpoints/dev.ts

@@ -0,0 +1,199 @@
+import * as t from 'ts-codec';
+import * as pgwire from '@powersync/service-jpgwire';
+import { errors, router, schema } from '@powersync/lib-services-framework';
+
+import * as util from '../../util/util-index.js';
+import { authDevUser, authUser, endpoint, issueDevToken, issueLegacyDevToken, issuePowerSyncToken } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+const AuthParams = t.object({
+  user: t.string,
+  password: t.string
+});
+
+// For legacy web client only. Remove soon.
+export const auth = routeDefinition({
+  path: '/auth.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
+  handler: async (payload) => {
+    const { user, password } = payload.params;
+    const config = payload.context.system.config;
+
+    if (config.dev.demo_auth == false || config.dev.demo_password == null) {
+      throw new errors.AuthorizationError(['Demo auth disabled']);
+    }
+
+    if (password == config.dev.demo_password) {
+      const token = await issueLegacyDevToken(payload.request, user, payload.context.system.config);
+      return { token, user_id: user, endpoint: endpoint(payload.request) };
+    } else {
+      throw new errors.AuthorizationError(['Authentication failed']);
+    }
+  }
+});
+
+export const auth2 = routeDefinition({
+  path: '/dev/auth.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
+  handler: async (payload) => {
+    const { user, password } = payload.params;
+    const config = payload.context.system.config;
+
+    if (config.dev.demo_auth == false || config.dev.demo_password == null) {
+      throw new errors.AuthorizationError(['Demo auth disabled']);
+    }
+
+    if (password == config.dev.demo_password) {
+      const token = await issueDevToken(payload.request, user, payload.context.system.config);
+      return { token, user_id: user };
+    } else {
+      throw new errors.AuthorizationError(['Authentication failed']);
+    }
+  }
+});
+
+const TokenParams = t.object({});
+
+export const token = routeDefinition({
+  path: '/dev/token.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(TokenParams, { allowAdditional: true }),
+  authorize: authDevUser,
+  handler: async (payload) => {
+    const { user_id } = payload.context;
+    const outToken = await issuePowerSyncToken(payload.request, user_id!, payload.context.system.config);
+    return { token: outToken, user_id: user_id, endpoint: endpoint(payload.request) };
+  }
+});
+
+const OpType = {
+  PUT: 'PUT',
+  PATCH: 'PATCH',
+  DELETE: 'DELETE'
+};
+
+const CrudEntry = t.object({
+  op: t.Enum(OpType),
+  type: t.string,
+  id: t.string,
+  op_id: t.number.optional(),
+  data: t.any.optional()
+});
+
+const CrudRequest = t.object({
+  data: t.array(CrudEntry),
+  write_checkpoint: t.boolean.optional()
+});
+
+export const crud = routeDefinition({
+  path: '/crud.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(CrudRequest, { allowAdditional: true }),
+  authorize: authUser,
+
+  handler: async (payload) => {
+    const { user_id, system } = payload.context;
+
+    const pool = system.requirePgPool();
+
+    if (!system.config.dev.crud_api) {
+      throw new Error('CRUD api disabled');
+    }
+
+    const params = payload.params;
+
+    let statements: pgwire.Statement[] = [];
+
+    // Implementation note:
+    // Postgres does automatic "assigment cast" for query literals,
+    // e.g. a string literal to uuid. However, the same doesn't apply
+    // to query parameters.
+    // To handle those automatically, we use `json_populate_record`
+    // to automatically cast to the correct types.
+
+    for (let op of params.data) {
+      const table = util.escapeIdentifier(op.type);
+      if (op.op == 'PUT') {
+        const data = op.data as Record<string, any>;
+        const with_id = { ...data, id: op.id };
+
+        const columnsEscaped = Object.keys(with_id).map(util.escapeIdentifier);
+        const columnsJoined = columnsEscaped.join(', ');
+
+        let updateClauses: string[] = [];
+
+        for (let key of Object.keys(data)) {
+          updateClauses.push(`${util.escapeIdentifier(key)} = EXCLUDED.${util.escapeIdentifier(key)}`);
+        }
+
+        const updateClause = updateClauses.length > 0 ? `DO UPDATE SET ${updateClauses.join(', ')}` : `DO NOTHING`;
+
+        const statement = `
+          WITH data_row AS (
+              SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          INSERT INTO ${table} (${columnsJoined})
+          SELECT ${columnsJoined} FROM data_row
+          ON CONFLICT(id) ${updateClause}`;
+
+        statements.push({
+          statement: statement,
+          params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
+        });
+      } else if (op.op == 'PATCH') {
+        const data = op.data as Record<string, any>;
+        const with_id = { ...data, id: op.id };
+
+        let updateClauses: string[] = [];
+
+        for (let key of Object.keys(data)) {
+          updateClauses.push(`${util.escapeIdentifier(key)} = data_row.${util.escapeIdentifier(key)}`);
+        }
+
+        const statement = `
+          WITH data_row AS (
+              SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          UPDATE ${table}
+          SET ${updateClauses.join(', ')}
+          FROM data_row
+          WHERE ${table}.id = data_row.id`;
+
+        statements.push({
+          statement: statement,
+          params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
+        });
+      } else if (op.op == 'DELETE') {
+        statements.push({
+          statement: `
+          WITH data_row AS (
+            SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          DELETE FROM ${table}
+          USING data_row
+          WHERE ${table}.id = data_row.id`,
+          params: [{ type: 'varchar', value: JSON.stringify({ id: op.id }) }]
+        });
+      }
+    }
+    await pool.query(...statements);
+
+    const storage = system.storage;
+    if (payload.params.write_checkpoint === true) {
+      const write_checkpoint = await util.createWriteCheckpoint(pool, storage, payload.context.user_id!);
+      return { write_checkpoint: String(write_checkpoint) };
+    } else if (payload.params.write_checkpoint === false) {
+      return {};
+    } else {
+      // Legacy
+      const checkpoint = await util.getClientCheckpoint(pool, storage);
+      return {
+        checkpoint
+      };
+    }
+  }
+});
+
+export const DEV_ROUTES = [auth, auth2, token, crud];

package/src/routes/endpoints/route-endpoints-index.ts

@@ -0,0 +1,6 @@
+export * from './admin.js';
+export * from './checkpointing.js';
+export * from './dev.js';
+export * from './socket-route.js';
+export * from './sync-rules.js';
+export * from './sync-stream.js';

package/src/routes/{socket-route.ts → endpoints/socket-route.ts}

@@ -1,14 +1,14 @@
+import { errors, logger, schema } from '@powersync/lib-services-framework';
+import { RequestParameters } from '@powersync/service-sync-rules';
 import { serialize } from 'bson';
-import { SyncParameters, normalizeTokenParameters } from '@powersync/service-sync-rules';
-import * as micro from '@journeyapps-platform/micro';
 
-import * as util from '@/util/util-index.js';
-import { streamResponse } from '../sync/sync.js';
+import { Metrics } from '../../metrics/Metrics.js';
+import { streamResponse } from '../../sync/sync.js';
+import * as util from '../../util/util-index.js';
+import { SocketRouteGenerator } from '../router-socket.js';
 import { SyncRoutes } from './sync-stream.js';
-import { SocketRouteGenerator } from './router-socket.js';
-import { Metrics } from '@/metrics/Metrics.js';
 
-export const sync_stream_reactive: SocketRouteGenerator = (router) =>
+export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
     authorize: ({ context }) => {
       return {
@@ -16,13 +16,13 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
         errors: ['Authentication required'].concat(context.token_errors ?? [])
       };
     },
-    validator: micro.schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
+    validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
     handler: async ({ context, params, responder, observer, initialN }) => {
       const { system } = context;
 
       if (system.closed) {
         responder.onError(
-          new micro.errors.JourneyError({
+          new errors.JourneyError({
             status: 503,
             code: 'SERVICE_UNAVAILABLE',
             description: 'Service temporarily unavailable'
@@ -34,17 +34,14 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
 
       const controller = new AbortController();
 
-      const syncParams: SyncParameters = normalizeTokenParameters(
-        context.token_payload?.parameters ?? {},
-        params.parameters ?? {}
-      );
+      const syncParams = new RequestParameters(context.token_payload!, params.parameters ?? {});
 
       const storage = system.storage;
       // Sanity check before we start the stream
       const cp = await storage.getActiveCheckpoint();
       if (!cp.hasSyncRules()) {
         responder.onError(
-          new micro.errors.JourneyError({
+          new errors.JourneyError({
             status: 500,
             code: 'NO_SYNC_RULES',
             description: 'No sync rules available'
@@ -122,8 +119,8 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
       } catch (ex) {
         // Convert to our standard form before responding.
         // This ensures the error can be serialized.
-        const error = new micro.errors.InternalServerError(ex);
-        micro.logger.error('Sync stream error', error);
+        const error = new errors.InternalServerError(ex);
+        logger.error('Sync stream error', error);
         responder.onError(error);
       } finally {
         responder.onComplete();

package/src/routes/endpoints/sync-rules.ts

@@ -0,0 +1,227 @@
+import * as t from 'ts-codec';
+import type { FastifyPluginAsync } from 'fastify';
+import * as pgwire from '@powersync/service-jpgwire';
+import { errors, router, schema } from '@powersync/lib-services-framework';
+import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
+
+import * as replication from '../../replication/replication-index.js';
+import { authApi } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+const DeploySyncRulesRequest = t.object({
+  content: t.string
+});
+
+export const yamlPlugin: FastifyPluginAsync = async (fastify) => {
+  fastify.addContentTypeParser('application/yaml', async (request, payload, _d) => {
+    const data: any[] = [];
+    for await (const chunk of payload) {
+      data.push(chunk);
+    }
+
+    request.params = { content: Buffer.concat(data).toString('utf8') };
+  });
+};
+
+/**
+ * Declares the plugin should be available on the same scope
+ * without requiring the `fastify-plugin` package as a dependency.
+ * https://fastify.dev/docs/latest/Reference/Plugins/#handle-the-scope
+ */
+//@ts-expect-error
+yamlPlugin[Symbol.for('skip-override')] = true;
+
+export const deploySyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/deploy',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  parse: true,
+  plugins: [yamlPlugin],
+  validator: schema.createTsCodecValidator(DeploySyncRulesRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    if (payload.context.system.config.sync_rules.present) {
+      // If sync rules are configured via the config, disable deploy via the API.
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'API_DISABLED',
+        description: 'Sync rules API disabled',
+        details: 'Use the management API to deploy sync rules'
+      });
+    }
+    const content = payload.params.content;
+
+    try {
+      SqlSyncRules.fromYaml(payload.params.content);
+    } catch (e) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'INVALID_SYNC_RULES',
+        description: 'Sync rules parsing failed',
+        details: e.message
+      });
+    }
+
+    const sync_rules = await payload.context.system.storage.updateSyncRules({
+      content: content
+    });
+
+    return {
+      slot_name: sync_rules.slot_name
+    };
+  }
+});
+
+const ValidateSyncRulesRequest = t.object({
+  content: t.string
+});
+
+export const validateSyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/validate',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  parse: true,
+  plugins: [yamlPlugin],
+  validator: schema.createTsCodecValidator(ValidateSyncRulesRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const content = payload.params.content;
+
+    const info = await debugSyncRules(payload.context.system.requirePgPool(), content);
+
+    return replyPrettyJson(info);
+  }
+});
+
+export const currentSyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/current',
+  method: router.HTTPMethod.GET,
+  authorize: authApi,
+  handler: async (payload) => {
+    const storage = payload.context.system.storage;
+    const sync_rules = await storage.getActiveSyncRulesContent();
+    if (!sync_rules) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'NO_SYNC_RULES',
+        description: 'No active sync rules'
+      });
+    }
+    const info = await debugSyncRules(payload.context.system.requirePgPool(), sync_rules.sync_rules_content);
+    const next = await storage.getNextSyncRulesContent();
+
+    const next_info = next
+      ? await debugSyncRules(payload.context.system.requirePgPool(), next.sync_rules_content)
+      : null;
+
+    const response = {
+      current: {
+        slot_name: sync_rules.slot_name,
+        content: sync_rules.sync_rules_content,
+        ...info
+      },
+      next:
+        next == null
+          ? null
+          : {
+              slot_name: next.slot_name,
+              content: next.sync_rules_content,
+              ...next_info
+            }
+    };
+
+    return replyPrettyJson({ data: response });
+  }
+});
+
+const ReprocessSyncRulesRequest = t.object({});
+
+export const reprocessSyncRules = routeDefinition({
+  path: '/api/sync-rules/v1/reprocess',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(ReprocessSyncRulesRequest),
+  handler: async (payload) => {
+    const storage = payload.context.system.storage;
+    const sync_rules = await storage.getActiveSyncRules();
+    if (sync_rules == null) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'NO_SYNC_RULES',
+        description: 'No active sync rules'
+      });
+    }
+
+    const new_rules = await storage.updateSyncRules({
+      content: sync_rules.sync_rules.content
+    });
+    return {
+      slot_name: new_rules.slot_name
+    };
+  }
+});
+
+export const SYNC_RULES_ROUTES = [validateSyncRules, deploySyncRules, reprocessSyncRules, currentSyncRules];
+
+function replyPrettyJson(payload: any) {
+  return new router.RouterResponse({
+    status: 200,
+    data: JSON.stringify(payload, null, 2) + '\n',
+    headers: { 'Content-Type': 'application/json' }
+  });
+}
+
+async function debugSyncRules(db: pgwire.PgClient, sync_rules: string) {
+  try {
+    const rules = SqlSyncRules.fromYaml(sync_rules);
+    const source_table_patterns = rules.getSourceTables();
+    const wc = new replication.WalConnection({
+      db: db,
+      sync_rules: rules
+    });
+    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+
+    return {
+      valid: true,
+      bucket_definitions: rules.bucket_descriptors.map((d) => {
+        let all_parameter_queries = [...d.parameter_queries.values()].flat();
+        let all_data_queries = [...d.data_queries.values()].flat();
+        return {
+          name: d.name,
+          bucket_parameters: d.bucket_parameters,
+          global_parameter_queries: d.global_parameter_queries.map((q) => {
+            return {
+              sql: q.sql
+            };
+          }),
+          parameter_queries: all_parameter_queries.map((q) => {
+            return {
+              sql: q.sql,
+              table: q.sourceTable,
+              input_parameters: q.input_parameters
+            };
+          }),
+
+          data_queries: all_data_queries.map((q) => {
+            return {
+              sql: q.sql,
+              table: q.sourceTable,
+              columns: q.columnOutputNames()
+            };
+          })
+        };
+      }),
+      source_tables: resolved_tables,
+      data_tables: rules.debugGetOutputTables()
+    };
+  } catch (e) {
+    if (e instanceof SyncRulesErrors) {
+      return {
+        valid: false,
+        errors: e.errors.map((e) => e.message)
+      };
+    }
+    return {
+      valid: false,
+      errors: [e.message]
+    };
+  }
+}

package/src/routes/endpoints/sync-stream.ts

@@ -0,0 +1,98 @@
+import { errors, logger, router, schema } from '@powersync/lib-services-framework';
+import { RequestParameters } from '@powersync/service-sync-rules';
+import { Readable } from 'stream';
+
+import * as sync from '../../sync/sync-index.js';
+import * as util from '../../util/util-index.js';
+
+import { Metrics } from '../../metrics/Metrics.js';
+import { authUser } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+export enum SyncRoutes {
+  STREAM = '/sync/stream'
+}
+
+export const syncStreamed = routeDefinition({
+  path: SyncRoutes.STREAM,
+  method: router.HTTPMethod.POST,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    if (system.closed) {
+      throw new errors.JourneyError({
+        status: 503,
+        code: 'SERVICE_UNAVAILABLE',
+        description: 'Service temporarily unavailable'
+      });
+    }
+
+    const params: util.StreamingSyncRequest = payload.params;
+    const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
+
+    const storage = system.storage;
+    // Sanity check before we start the stream
+    const cp = await storage.getActiveCheckpoint();
+    if (!cp.hasSyncRules()) {
+      throw new errors.JourneyError({
+        status: 500,
+        code: 'NO_SYNC_RULES',
+        description: 'No sync rules available'
+      });
+    }
+    const controller = new AbortController();
+    try {
+      Metrics.getInstance().concurrent_connections.add(1);
+      const stream = Readable.from(
+        sync.transformToBytesTracked(
+          sync.ndjson(
+            sync.streamResponse({
+              storage,
+              params,
+              syncParams,
+              token: payload.context.token_payload!,
+              signal: controller.signal
+            })
+          )
+        ),
+        { objectMode: false, highWaterMark: 16 * 1024 }
+      );
+
+      const deregister = system.addStopHandler(() => {
+        // This error is not currently propagated to the client
+        controller.abort();
+        stream.destroy(new Error('Shutting down system'));
+      });
+      stream.on('close', () => {
+        deregister();
+      });
+
+      stream.on('error', (error) => {
+        controller.abort();
+        // Note: This appears as a 200 response in the logs.
+        if (error.message != 'Shutting down system') {
+          logger.error('Streaming sync request failed', error);
+        }
+      });
+
+      return new router.RouterResponse({
+        status: 200,
+        headers: {
+          'Content-Type': 'application/x-ndjson'
+        },
+        data: stream,
+        afterSend: async () => {
+          controller.abort();
+          Metrics.getInstance().concurrent_connections.add(-1);
+        }
+      });
+    } catch (ex) {
+      controller.abort();
+      Metrics.getInstance().concurrent_connections.add(-1);
+    }
+  }
+});
+
+export const SYNC_STREAM_ROUTES = [syncStreamed];

package/src/routes/hooks.ts

@@ -0,0 +1,45 @@
+import type fastify from 'fastify';
+import a from 'async';
+import { logger } from '@powersync/lib-services-framework';
+
+export type CreateRequestQueueParams = {
+  max_queue_depth: number;
+  concurrency: number;
+};
+
+/**
+ * Creates a request queue which limits the amount of concurrent connections which
+ * are active at any time.
+ */
+export const createRequestQueueHook = (params: CreateRequestQueueParams): fastify.onRequestHookHandler => {
+  const request_queue = a.queue<() => Promise<void>>((event, done) => {
+    event().finally(done);
+  }, params.concurrency);
+
+  return (request, reply, next) => {
+    if (
+      (params.max_queue_depth == 0 && request_queue.running() == params.concurrency) ||
+      (params.max_queue_depth > 0 && request_queue.length() >= params.max_queue_depth)
+    ) {
+      logger.warn(`${request.method} ${request.url}`, {
+        status: 429,
+        method: request.method,
+        path: request.url,
+        queue_overflow: true
+      });
+      return reply.status(429).send();
+    }
+
+    const finished = new Promise<void>((resolve) => {
+      reply.then(
+        () => resolve(),
+        () => resolve()
+      );
+    });
+
+    request_queue.push(() => {
+      next();
+      return finished;
+    });
+  };
+};