@powersync/service-core 0.2.2 → 0.4.0

package/src/routes/dev.ts

@@ -1,194 +0,0 @@
-import * as t from 'ts-codec';
-import * as micro from '@journeyapps-platform/micro';
-import * as pgwire from '@powersync/service-jpgwire';
-
-import * as util from '@/util/util-index.js';
-import { authDevUser, authUser, endpoint, issueDevToken, issueLegacyDevToken, issuePowerSyncToken } from './auth.js';
-import { RouteGenerator } from './router.js';
-
-const AuthParams = t.object({
-  user: t.string,
-  password: t.string
-});
-
-// For legacy web client only. Remove soon.
-export const auth: RouteGenerator = (router) =>
-  router.post('/auth.json', {
-    validator: micro.schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
-    handler: async (payload) => {
-      const { user, password } = payload.params;
-      const config = payload.context.system.config;
-
-      if (config.dev.demo_auth == false || config.dev.demo_password == null) {
-        throw new micro.errors.AuthorizationError(['Demo auth disabled']);
-      }
-
-      if (password == config.dev.demo_password) {
-        const token = await issueLegacyDevToken(payload.request, user, payload.context.system.config);
-        return { token, user_id: user, endpoint: endpoint(payload.request) };
-      } else {
-        throw new micro.errors.AuthorizationError(['Authentication failed']);
-      }
-    }
-  });
-
-export const auth2: RouteGenerator = (router) =>
-  router.post('/dev/auth.json', {
-    validator: micro.schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
-    handler: async (payload) => {
-      const { user, password } = payload.params;
-      const config = payload.context.system.config;
-
-      if (config.dev.demo_auth == false || config.dev.demo_password == null) {
-        throw new micro.errors.AuthorizationError(['Demo auth disabled']);
-      }
-
-      if (password == config.dev.demo_password) {
-        const token = await issueDevToken(payload.request, user, payload.context.system.config);
-        return { token, user_id: user };
-      } else {
-        throw new micro.errors.AuthorizationError(['Authentication failed']);
-      }
-    }
-  });
-
-const TokenParams = t.object({});
-
-export const token: RouteGenerator = (router) =>
-  router.post('/dev/token.json', {
-    validator: micro.schema.createTsCodecValidator(TokenParams, { allowAdditional: true }),
-    authorize: authDevUser,
-    handler: async (payload) => {
-      const { user_id } = payload.context;
-      const outToken = await issuePowerSyncToken(payload.request, user_id!, payload.context.system.config);
-      return { token: outToken, user_id: user_id, endpoint: endpoint(payload.request) };
-    }
-  });
-
-const OpType = {
-  PUT: 'PUT',
-  PATCH: 'PATCH',
-  DELETE: 'DELETE'
-};
-
-const CrudEntry = t.object({
-  op: t.Enum(OpType),
-  type: t.string,
-  id: t.string,
-  op_id: t.number.optional(),
-  data: t.any.optional()
-});
-
-const CrudRequest = t.object({
-  data: t.array(CrudEntry),
-  write_checkpoint: t.boolean.optional()
-});
-export const crud: RouteGenerator = (router) =>
-  router.post('/crud.json', {
-    validator: micro.schema.createTsCodecValidator(CrudRequest, { allowAdditional: true }),
-    authorize: authUser,
-
-    handler: async (payload) => {
-      const { user_id, system } = payload.context;
-
-      const pool = system.requirePgPool();
-
-      if (!system.config.dev.crud_api) {
-        throw new Error('CRUD api disabled');
-      }
-
-      const params = payload.params;
-
-      let statements: pgwire.Statement[] = [];
-
-      // Implementation note:
-      // Postgres does automatic "assigment cast" for query literals,
-      // e.g. a string literal to uuid. However, the same doesn't apply
-      // to query parameters.
-      // To handle those automatically, we use `json_populate_record`
-      // to automatically cast to the correct types.
-
-      for (let op of params.data) {
-        const table = util.escapeIdentifier(op.type);
-        if (op.op == 'PUT') {
-          const data = op.data as Record<string, any>;
-          const with_id = { ...data, id: op.id };
-
-          const columnsEscaped = Object.keys(with_id).map(util.escapeIdentifier);
-          const columnsJoined = columnsEscaped.join(', ');
-
-          let updateClauses: string[] = [];
-
-          for (let key of Object.keys(data)) {
-            updateClauses.push(`${util.escapeIdentifier(key)} = EXCLUDED.${util.escapeIdentifier(key)}`);
-          }
-
-          const updateClause = updateClauses.length > 0 ? `DO UPDATE SET ${updateClauses.join(', ')}` : `DO NOTHING`;
-
-          const statement = `
-          WITH data_row AS (
-              SELECT (json_populate_record(null::${table}, $1::json)).*
-          )
-          INSERT INTO ${table} (${columnsJoined})
-          SELECT ${columnsJoined} FROM data_row
-          ON CONFLICT(id) ${updateClause}`;
-
-          statements.push({
-            statement: statement,
-            params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
-          });
-        } else if (op.op == 'PATCH') {
-          const data = op.data as Record<string, any>;
-          const with_id = { ...data, id: op.id };
-
-          let updateClauses: string[] = [];
-
-          for (let key of Object.keys(data)) {
-            updateClauses.push(`${util.escapeIdentifier(key)} = data_row.${util.escapeIdentifier(key)}`);
-          }
-
-          const statement = `
-          WITH data_row AS (
-              SELECT (json_populate_record(null::${table}, $1::json)).*
-          )
-          UPDATE ${table}
-          SET ${updateClauses.join(', ')}
-          FROM data_row
-          WHERE ${table}.id = data_row.id`;
-
-          statements.push({
-            statement: statement,
-            params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
-          });
-        } else if (op.op == 'DELETE') {
-          statements.push({
-            statement: `
-          WITH data_row AS (
-            SELECT (json_populate_record(null::${table}, $1::json)).*
-          )
-          DELETE FROM ${table}
-          USING data_row
-          WHERE ${table}.id = data_row.id`,
-            params: [{ type: 'varchar', value: JSON.stringify({ id: op.id }) }]
-          });
-        }
-      }
-      await pool.query(...statements);
-
-      const storage = system.storage;
-      if (payload.params.write_checkpoint === true) {
-        const write_checkpoint = await util.createWriteCheckpoint(pool, storage, payload.context.user_id!);
-        return { write_checkpoint: String(write_checkpoint) };
-      } else if (payload.params.write_checkpoint === false) {
-        return {};
-      } else {
-        // Legacy
-        const checkpoint = await util.getClientCheckpoint(pool, storage);
-        return {
-          checkpoint
-        };
-      }
-    }
-  });
-
-export const dev_routes = [auth, auth2, token, crud];

package/src/routes/route-generators.ts

@@ -1,39 +0,0 @@
-import * as micro from '@journeyapps-platform/micro';
-
-import { Context, RouteGenerator } from './router.js';
-import { admin_routes } from './admin.js';
-import { writeCheckpoint, writeCheckpoint2 } from './checkpointing.js';
-import { dev_routes } from './dev.js';
-import { syncRulesRoutes } from './sync-rules.js';
-import { IReactiveStream, ReactiveSocketRouter } from '@powersync/service-rsocket-router';
-import { sync_stream_reactive } from './socket-route.js';
-import { syncStreamed } from './sync-stream.js';
-
-/**
- * Generates router endpoints using the specified router instance
- */
-export const generateHTTPRoutes = (router: micro.fastify.FastifyRouter<Context>): micro.router.Route[] => {
-  const generators: RouteGenerator[] = [
-    ...dev_routes,
-    writeCheckpoint,
-    writeCheckpoint2,
-    ...syncRulesRoutes,
-    ...admin_routes
-  ];
-
-  return generators.map((generateRoute) => generateRoute(router));
-};
-
-/**
- * Generates route endpoint for HTTP sync streaming
- */
-export const generateHTTPStreamRoutes = (router: micro.fastify.FastifyRouter<Context>): micro.router.Route[] => {
-  return [syncStreamed].map((generateRoute) => generateRoute(router));
-};
-
-/**
- *  Generates socket router endpoints using the specified router instance
- */
-export const generateSocketRoutes = (router: ReactiveSocketRouter<Context>): IReactiveStream[] => {
-  return [sync_stream_reactive].map((generateRoute) => generateRoute(router));
-};

package/src/routes/sync-rules.ts

@@ -1,210 +0,0 @@
-import * as t from 'ts-codec';
-import { FastifyPluginAsync, FastifyReply } from 'fastify';
-import * as micro from '@journeyapps-platform/micro';
-import * as pgwire from '@powersync/service-jpgwire';
-import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
-
-import * as replication from '@/replication/replication-index.js';
-import { authApi } from './auth.js';
-import { RouteGenerator } from './router.js';
-
-const DeploySyncRulesRequest = t.object({
-  content: t.string
-});
-
-const yamlPlugin: FastifyPluginAsync = async (fastify) => {
-  fastify.addContentTypeParser('application/yaml', async (request, payload, _d) => {
-    const data = await micro.streaming.drain(payload);
-
-    request.params = { content: Buffer.concat(data).toString('utf8') };
-  });
-};
-
-export const deploySyncRules: RouteGenerator = (router) =>
-  router.post('/api/sync-rules/v1/deploy', {
-    authorize: authApi,
-    parse: true,
-    plugins: [yamlPlugin],
-    validator: micro.schema.createTsCodecValidator(DeploySyncRulesRequest, { allowAdditional: true }),
-    handler: async (payload) => {
-      if (payload.context.system.config.sync_rules.present) {
-        // If sync rules are configured via the config, disable deploy via the API.
-        throw new micro.errors.JourneyError({
-          status: 422,
-          code: 'API_DISABLED',
-          description: 'Sync rules API disabled',
-          details: 'Use the management API to deploy sync rules'
-        });
-      }
-      const content = payload.params.content;
-
-      try {
-        SqlSyncRules.fromYaml(payload.params.content);
-      } catch (e) {
-        throw new micro.errors.JourneyError({
-          status: 422,
-          code: 'INVALID_SYNC_RULES',
-          description: 'Sync rules parsing failed',
-          details: e.message
-        });
-      }
-
-      const sync_rules = await payload.context.system.storage.updateSyncRules({
-        content: content
-      });
-
-      return {
-        slot_name: sync_rules.slot_name
-      };
-    }
-  });
-
-const ValidateSyncRulesRequest = t.object({
-  content: t.string
-});
-
-export const validateSyncRules: RouteGenerator = (router) =>
-  router.post('/api/sync-rules/v1/validate', {
-    authorize: authApi,
-    parse: true,
-    plugins: [yamlPlugin],
-    validator: micro.schema.createTsCodecValidator(ValidateSyncRulesRequest, { allowAdditional: true }),
-    handler: async (payload) => {
-      const content = payload.params.content;
-
-      const info = await debugSyncRules(payload.context.system.requirePgPool(), content);
-
-      replyPrettyJson(payload.reply, info);
-    }
-  });
-
-export const currentSyncRules: RouteGenerator = (router) =>
-  router.get('/api/sync-rules/v1/current', {
-    authorize: authApi,
-    handler: async (payload) => {
-      const storage = payload.context.system.storage;
-      const sync_rules = await storage.getActiveSyncRulesContent();
-      if (!sync_rules) {
-        throw new micro.errors.JourneyError({
-          status: 422,
-          code: 'NO_SYNC_RULES',
-          description: 'No active sync rules'
-        });
-      }
-      const info = await debugSyncRules(payload.context.system.requirePgPool(), sync_rules.sync_rules_content);
-      const next = await storage.getNextSyncRulesContent();
-
-      const next_info = next
-        ? await debugSyncRules(payload.context.system.requirePgPool(), next.sync_rules_content)
-        : null;
-
-      const response = {
-        current: {
-          slot_name: sync_rules.slot_name,
-          content: sync_rules.sync_rules_content,
-          ...info
-        },
-        next:
-          next == null
-            ? null
-            : {
-                slot_name: next.slot_name,
-                content: next.sync_rules_content,
-                ...next_info
-              }
-      };
-      replyPrettyJson(payload.reply, { data: response });
-    }
-  });
-
-const ReprocessSyncRulesRequest = t.object({});
-
-export const reprocessSyncRules: RouteGenerator = (router) =>
-  router.post('/api/sync-rules/v1/reprocess', {
-    authorize: authApi,
-    validator: micro.schema.createTsCodecValidator(ReprocessSyncRulesRequest),
-    handler: async (payload) => {
-      const storage = payload.context.system.storage;
-      const sync_rules = await storage.getActiveSyncRules();
-      if (sync_rules == null) {
-        throw new micro.errors.JourneyError({
-          status: 422,
-          code: 'NO_SYNC_RULES',
-          description: 'No active sync rules'
-        });
-      }
-
-      const new_rules = await storage.updateSyncRules({
-        content: sync_rules.sync_rules.content
-      });
-      return {
-        slot_name: new_rules.slot_name
-      };
-    }
-  });
-
-export const syncRulesRoutes = [validateSyncRules, deploySyncRules, reprocessSyncRules, currentSyncRules];
-
-function replyPrettyJson(reply: FastifyReply, payload: any) {
-  reply
-    .status(200)
-    .header('Content-Type', 'application/json')
-    .send(JSON.stringify(payload, null, 2) + '\n');
-}
-
-async function debugSyncRules(db: pgwire.PgClient, sync_rules: string) {
-  try {
-    const rules = SqlSyncRules.fromYaml(sync_rules);
-    const source_table_patterns = rules.getSourceTables();
-    const wc = new replication.WalConnection({
-      db: db,
-      sync_rules: rules
-    });
-    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
-
-    return {
-      valid: true,
-      bucket_definitions: rules.bucket_descriptors.map((d) => {
-        let all_parameter_queries = [...d.parameter_queries.values()].flat();
-        let all_data_queries = [...d.data_queries.values()].flat();
-        return {
-          name: d.name,
-          bucket_parameters: d.bucket_parameters,
-          global_parameter_queries: d.global_parameter_queries.map((q) => {
-            return {
-              sql: q.sql
-            };
-          }),
-          parameter_queries: all_parameter_queries.map((q) => {
-            return {
-              sql: q.sql,
-              table: q.sourceTable,
-              input_parameters: q.input_parameters
-            };
-          }),
-
-          data_queries: all_data_queries.map((q) => {
-            return {
-              sql: q.sql,
-              table: q.sourceTable,
-              columns: q.columnOutputNames()
-            };
-          })
-        };
-      }),
-      source_tables: resolved_tables,
-      data_tables: rules.debugGetOutputTables()
-    };
-  } catch (e) {
-    if (e instanceof SyncRulesErrors) {
-      return {
-        valid: false,
-        errors: e.errors.map((e) => e.message)
-      };
-    }
-    return {
-      valid: false,
-      errors: [e.message]
-    };
-  }
-}

package/src/routes/sync-stream.ts

@@ -1,95 +0,0 @@
-import { Readable } from 'stream';
-import * as micro from '@journeyapps-platform/micro';
-import { SyncParameters, normalizeTokenParameters } from '@powersync/service-sync-rules';
-
-import * as sync from '@/sync/sync-index.js';
-import * as util from '@/util/util-index.js';
-
-import { authUser } from './auth.js';
-import { RouteGenerator } from './router.js';
-import { Metrics } from '@/metrics/Metrics.js';
-
-export enum SyncRoutes {
-  STREAM = '/sync/stream'
-}
-
-export const syncStreamed: RouteGenerator = (router) =>
-  router.post(SyncRoutes.STREAM, {
-    authorize: authUser,
-    validator: micro.schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
-    handler: async (payload) => {
-      const userId = payload.context.user_id!;
-      const system = payload.context.system;
-
-      if (system.closed) {
-        throw new micro.errors.JourneyError({
-          status: 503,
-          code: 'SERVICE_UNAVAILABLE',
-          description: 'Service temporarily unavailable'
-        });
-      }
-
-      const params: util.StreamingSyncRequest = payload.params;
-      const syncParams: SyncParameters = normalizeTokenParameters(
-        payload.context.token_payload!.parameters ?? {},
-        payload.params.parameters ?? {}
-      );
-
-      const storage = system.storage;
-      // Sanity check before we start the stream
-      const cp = await storage.getActiveCheckpoint();
-      if (!cp.hasSyncRules()) {
-        throw new micro.errors.JourneyError({
-          status: 500,
-          code: 'NO_SYNC_RULES',
-          description: 'No sync rules available'
-        });
-      }
-
-      const res = payload.reply;
-
-      res.status(200).header('Content-Type', 'application/x-ndjson');
-
-      const controller = new AbortController();
-      try {
-        Metrics.getInstance().concurrent_connections.add(1);
-        const stream = Readable.from(
-          sync.transformToBytesTracked(
-            sync.ndjson(
-              sync.streamResponse({
-                storage,
-                params,
-                syncParams,
-                token: payload.context.token_payload!,
-                signal: controller.signal
-              })
-            )
-          ),
-          { objectMode: false, highWaterMark: 16 * 1024 }
-        );
-
-        const deregister = system.addStopHandler(() => {
-          // This error is not currently propagated to the client
-          controller.abort();
-          stream.destroy(new Error('Shutting down system'));
-        });
-        stream.on('close', () => {
-          deregister();
-        });
-
-        stream.on('error', (error) => {
-          controller.abort();
-          // Note: This appears as a 200 response in the logs.
-          if (error.message != 'Shutting down system') {
-            micro.logger.error('Streaming sync request failed', error);
-          }
-        });
-        await res.send(stream);
-      } finally {
-        controller.abort();
-        Metrics.getInstance().concurrent_connections.add(-1);
-        // Prevent double-send
-        res.hijack();
-      }
-    }
-  });