@powersync/service-core 0.0.2

package/src/routes/admin.ts

@@ -0,0 +1,229 @@
+import * as micro from '@journeyapps-platform/micro';
+import { SqlSyncRules, SqliteValue, StaticSchema, isJsonValue, toSyncRulesValue } from '@powersync/service-sync-rules';
+import { internal_routes } from '@powersync/service-types';
+
+import * as api from '@/api/api-index.js';
+import * as util from '@/util/util-index.js';
+
+import { RouteGenerator } from './router.js';
+import { PersistedSyncRulesContent } from '../storage/BucketStorage.js';
+import { authApi } from './auth.js';
+
+const demoCredentials: RouteGenerator = (router) =>
+  router.post('/api/admin/v1/demo-credentials', {
+    authorize: authApi,
+    validator: micro.schema.createTsCodecValidator(internal_routes.DemoCredentialsRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const connection = payload.context.system.config.connection;
+      if (connection == null || !connection.demo_database) {
+        return internal_routes.DemoCredentialsResponse.encode({});
+      }
+
+      const uri = util.buildDemoPgUri(connection);
+      return internal_routes.DemoCredentialsResponse.encode({
+        credentials: {
+          postgres_uri: uri
+        }
+      });
+    }
+  });
+
+export const executeSql: RouteGenerator = (router) =>
+  router.post('/api/admin/v1/execute-sql', {
+    authorize: authApi,
+    validator: micro.schema.createTsCodecValidator(internal_routes.ExecuteSqlRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const connection = payload.context.system.config.connection;
+      if (connection == null || !connection.debug_api) {
+        return internal_routes.ExecuteSqlResponse.encode({
+          results: {
+            columns: [],
+            rows: []
+          },
+          success: false,
+          error: 'SQL querying is not enabled'
+        });
+      }
+
+      const pool = payload.context.system.requirePgPool();
+
+      const { query, args } = payload.params.sql;
+
+      try {
+        const result = await pool.query({
+          statement: query,
+          params: args.map(util.autoParameter)
+        });
+
+        return internal_routes.ExecuteSqlResponse.encode({
+          success: true,
+          results: {
+            columns: result.columns.map((c) => c.name),
+            rows: result.rows.map((row) => {
+              return row.map((value) => mapColumnValue(toSyncRulesValue(value)));
+            })
+          }
+        });
+      } catch (e) {
+        return internal_routes.ExecuteSqlResponse.encode({
+          results: {
+            columns: [],
+            rows: []
+          },
+          success: false,
+          error: e.message
+        });
+      }
+    }
+  });
+
+export const diagnostics: RouteGenerator = (router) =>
+  router.post('/api/admin/v1/diagnostics', {
+    authorize: authApi,
+    validator: micro.schema.createTsCodecValidator(internal_routes.DiagnosticsRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const include_content = payload.params.sync_rules_content ?? false;
+      const system = payload.context.system;
+
+      const status = await api.getConnectionStatus(system);
+      if (status == null) {
+        return internal_routes.DiagnosticsResponse.encode({
+          connections: []
+        });
+      }
+
+      const { storage } = system;
+      const active = await storage.getActiveSyncRulesContent();
+      const next = await storage.getNextSyncRulesContent();
+
+      const active_status = await api.getSyncRulesStatus(active, system, {
+        include_content,
+        check_connection: status.connected,
+        live_status: true
+      });
+
+      const next_status = await api.getSyncRulesStatus(next, system, {
+        include_content,
+        check_connection: status.connected,
+        live_status: true
+      });
+
+      return internal_routes.DiagnosticsResponse.encode({
+        connections: [status],
+        active_sync_rules: active_status,
+        deploying_sync_rules: next_status
+      });
+    }
+  });
+
+export const getSchema: RouteGenerator = (router) =>
+  router.post('/api/admin/v1/schema', {
+    authorize: authApi,
+    validator: micro.schema.createTsCodecValidator(internal_routes.GetSchemaRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const system = payload.context.system;
+
+      return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(system));
+    }
+  });
+
+export const reprocess: RouteGenerator = (router) =>
+  router.post('/api/admin/v1/reprocess', {
+    authorize: authApi,
+    validator: micro.schema.createTsCodecValidator(internal_routes.ReprocessRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const system = payload.context.system;
+
+      const storage = system.storage;
+      const next = await storage.getNextSyncRules();
+      if (next != null) {
+        throw new Error(`Busy processing sync rules - cannot reprocess`);
+      }
+
+      const active = await storage.getActiveSyncRules();
+      if (active == null) {
+        throw new micro.errors.JourneyError({
+          status: 422,
+          code: 'NO_SYNC_RULES',
+          description: 'No active sync rules'
+        });
+      }
+
+      const new_rules = await storage.updateSyncRules({
+        content: active.sync_rules.content
+      });
+
+      return internal_routes.ReprocessResponse.encode({
+        connections: [
+          {
+            tag: system.config.connection!.tag,
+            id: system.config.connection!.id,
+            slot_name: new_rules.slot_name
+          }
+        ]
+      });
+    }
+  });
+
+export const validate: RouteGenerator = (router) =>
+  router.post('/api/admin/v1/validate', {
+    authorize: authApi,
+    validator: micro.schema.createTsCodecValidator(internal_routes.ValidateRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const system = payload.context.system;
+
+      const content = payload.params.sync_rules;
+
+      const schemaData = await api.getConnectionsSchema(system);
+      const schema = new StaticSchema(schemaData.connections);
+
+      const sync_rules: PersistedSyncRulesContent = {
+        // Dummy values
+        id: 0,
+        slot_name: '',
+
+        parsed() {
+          return {
+            ...this,
+            sync_rules: SqlSyncRules.fromYaml(content, { throwOnError: false, schema })
+          };
+        },
+        sync_rules_content: content,
+        async lock() {
+          throw new Error('Lock not implemented');
+        }
+      };
+
+      const connectionStatus = await api.getConnectionStatus(system);
+      if (connectionStatus == null) {
+        return internal_routes.ValidateResponse.encode({
+          errors: [{ level: 'fatal', message: 'No connection configured' }],
+          connections: []
+        });
+      }
+
+      const status = (await api.getSyncRulesStatus(sync_rules, system, {
+        include_content: false,
+        check_connection: connectionStatus?.connected,
+        live_status: false
+      }))!;
+
+      if (connectionStatus == null) {
+        status.errors.push({ level: 'fatal', message: 'No connection configured' });
+      }
+
+      return internal_routes.ValidateResponse.encode(status);
+    }
+  });
+
+function mapColumnValue(value: SqliteValue) {
+  if (typeof value == 'bigint') {
+    return Number(value);
+  } else if (isJsonValue(value)) {
+    return value;
+  } else {
+    return null;
+  }
+}
+
+export const admin_routes = [demoCredentials, executeSql, diagnostics, getSchema, reprocess, validate];

package/src/routes/auth.ts

@@ -0,0 +1,209 @@
+import * as micro from '@journeyapps-platform/micro';
+import { FastifyRequest } from 'fastify';
+import * as jose from 'jose';
+
+import * as auth from '@/auth/auth-index.js';
+import * as util from '@/util/util-index.js';
+import { Context } from './router.js';
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+
+export function endpoint(req: FastifyRequest) {
+  const protocol = req.headers['x-forwarded-proto'] ?? req.protocol;
+  const host = req.hostname;
+  return `${protocol}://${host}`;
+}
+
+function devAudience(req: FastifyRequest): string {
+  return `${endpoint(req)}/dev`;
+}
+
+/**
+ * @deprecated
+ *
+ * Will be replaced by temporary tokens issued by PowerSync Management service.
+ */
+export async function issueDevToken(req: FastifyRequest, user_id: string, config: util.ResolvedPowerSyncConfig) {
+  const iss = devAudience(req);
+  const aud = devAudience(req);
+
+  const key = config.dev.dev_key;
+  if (key == null) {
+    throw new Error('Auth disabled');
+  }
+
+  return await new jose.SignJWT({})
+    .setProtectedHeader({ alg: key.source.alg!, kid: key.kid })
+    .setSubject(user_id)
+    .setIssuedAt()
+    .setIssuer(iss)
+    .setAudience(aud)
+    .setExpirationTime('30d')
+    .sign(key.key);
+}
+
+/** @deprecated */
+export async function issueLegacyDevToken(req: FastifyRequest, user_id: string, config: util.ResolvedPowerSyncConfig) {
+  const iss = devAudience(req);
+  const aud = config.jwt_audiences[0];
+
+  const key = config.dev.dev_key;
+  if (key == null || aud == null) {
+    throw new Error('Auth disabled');
+  }
+
+  return await new jose.SignJWT({})
+    .setProtectedHeader({ alg: key.source.alg!, kid: key.kid })
+    .setSubject(user_id)
+    .setIssuedAt()
+    .setIssuer(iss)
+    .setAudience(aud)
+    .setExpirationTime('60m')
+    .sign(key.key);
+}
+
+export async function issuePowerSyncToken(req: FastifyRequest, user_id: string, config: util.ResolvedPowerSyncConfig) {
+  const iss = devAudience(req);
+  const aud = config.jwt_audiences[0];
+  const key = config.dev.dev_key;
+  if (key == null || aud == null) {
+    throw new Error('Auth disabled');
+  }
+
+  const jwt = await new jose.SignJWT({})
+    .setProtectedHeader({ alg: key.source.alg!, kid: key.kid })
+    .setSubject(user_id)
+    .setIssuedAt()
+    .setIssuer(iss)
+    .setAudience(aud)
+    .setExpirationTime('5m')
+    .sign(key.key);
+  return jwt;
+}
+
+export function getTokenFromHeader(authHeader: string = ''): string | null {
+  const tokenMatch = /^(Token|Bearer) (\S+)$/.exec(authHeader);
+  if (!tokenMatch) {
+    return null;
+  }
+  const token = tokenMatch[2];
+  return token ?? null;
+}
+
+export const authUser = async (payload: micro.fastify.FastifyHandlerPayload<any, Context>) => {
+  return authorizeUser(payload.context, payload.request.headers.authorization);
+};
+
+export async function authorizeUser(context: Context, authHeader: string = '') {
+  const token = getTokenFromHeader(authHeader);
+  if (token == null) {
+    return {
+      authorized: false,
+      errors: ['Authentication required']
+    };
+  }
+
+  const { context: tokenContext, errors } = await generateContext(context.system, token);
+
+  if (!tokenContext) {
+    return {
+      authorized: false,
+      errors
+    };
+  }
+
+  Object.assign(context, tokenContext);
+  return { authorized: true };
+}
+
+export async function generateContext(system: CorePowerSyncSystem, token: string) {
+  const config = system.config;
+
+  let tokenPayload: auth.JwtPayload;
+  try {
+    const maxAge = config.token_max_expiration;
+    tokenPayload = await system.client_keystore.verifyJwt(token, {
+      defaultAudiences: config.jwt_audiences,
+      maxAge: maxAge
+    });
+    return {
+      context: {
+        user_id: tokenPayload.sub,
+        token_payload: tokenPayload
+      }
+    };
+  } catch (err) {
+    return {
+      context: null,
+      errors: [err.message]
+    };
+  }
+}
+
+/**
+ * @deprecated
+ */
+export const authDevUser = async (payload: micro.fastify.FastifyHandlerPayload<any, Context>) => {
+  const context = payload.context;
+  const token = getTokenFromHeader(payload.request.headers.authorization);
+  if (!context.system.config.dev.demo_auth) {
+    return {
+      authorized: false,
+      errors: ['Authentication disabled']
+    };
+  }
+  if (token == null) {
+    return {
+      authorized: false,
+      errors: ['Authentication required']
+    };
+  }
+
+  // Different from the configured audience.
+  // Should also not be changed by keys
+  const audience = [devAudience(payload.request)];
+
+  let tokenPayload: auth.JwtPayload;
+  try {
+    tokenPayload = await context.system.dev_client_keystore.verifyJwt(token, {
+      defaultAudiences: audience,
+      maxAge: '31d'
+    });
+  } catch (err) {
+    return {
+      authorized: false,
+      errors: [err.message]
+    };
+  }
+
+  payload.context.user_id = tokenPayload.sub;
+  return { authorized: true };
+};
+
+export const authApi = (payload: micro.fastify.FastifyHandlerPayload<any, Context>) => {
+  const context = payload.context;
+  const api_keys = context.system.config.api_tokens;
+  if (api_keys.length == 0) {
+    return {
+      authorized: false,
+      errors: ['Authentication disabled']
+    };
+  }
+  const auth = payload.request.headers.authorization ?? '';
+
+  const tokenMatch = /^(Token|Bearer) (\S+)$/.exec(auth);
+  if (!tokenMatch) {
+    return {
+      authorized: false,
+      errors: ['Authentication required']
+    };
+  }
+  const token = tokenMatch[2];
+  if (api_keys.includes(token)) {
+    return { authorized: true };
+  } else {
+    return {
+      authorized: false,
+      errors: ['Authentication failed']
+    };
+  }
+};

package/src/routes/checkpointing.ts

@@ -0,0 +1,38 @@
+import * as t from 'ts-codec';
+import * as micro from '@journeyapps-platform/micro';
+
+import * as util from '@/util/util-index.js';
+
+import { authUser } from './auth.js';
+import { RouteGenerator } from './router.js';
+
+const WriteCheckpointRequest = t.object({});
+
+export const writeCheckpoint: RouteGenerator = (router) =>
+  router.get('/write-checkpoint.json', {
+    authorize: authUser,
+    validator: micro.schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const system = payload.context.system;
+      const storage = system.storage;
+
+      const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
+      return {
+        checkpoint
+      };
+    }
+  });
+
+export const writeCheckpoint2: RouteGenerator = (router) =>
+  router.get('/write-checkpoint2.json', {
+    authorize: authUser,
+    validator: micro.schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const { user_id, system } = payload.context;
+      const storage = system.storage;
+      const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, user_id!);
+      return {
+        write_checkpoint: String(write_checkpoint)
+      };
+    }
+  });

package/src/routes/dev.ts

@@ -0,0 +1,194 @@
+import * as t from 'ts-codec';
+import * as micro from '@journeyapps-platform/micro';
+import * as pgwire from '@powersync/service-jpgwire';
+
+import * as util from '@/util/util-index.js';
+import { authDevUser, authUser, endpoint, issueDevToken, issueLegacyDevToken, issuePowerSyncToken } from './auth.js';
+import { RouteGenerator } from './router.js';
+
+const AuthParams = t.object({
+  user: t.string,
+  password: t.string
+});
+
+// For legacy web client only. Remove soon.
+export const auth: RouteGenerator = (router) =>
+  router.post('/auth.json', {
+    validator: micro.schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
+    handler: async (payload) => {
+      const { user, password } = payload.params;
+      const config = payload.context.system.config;
+
+      if (config.dev.demo_auth == false || config.dev.demo_password == null) {
+        throw new micro.errors.AuthorizationError(['Demo auth disabled']);
+      }
+
+      if (password == config.dev.demo_password) {
+        const token = await issueLegacyDevToken(payload.request, user, payload.context.system.config);
+        return { token, user_id: user, endpoint: endpoint(payload.request) };
+      } else {
+        throw new micro.errors.AuthorizationError(['Authentication failed']);
+      }
+    }
+  });
+
+export const auth2: RouteGenerator = (router) =>
+  router.post('/dev/auth.json', {
+    validator: micro.schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
+    handler: async (payload) => {
+      const { user, password } = payload.params;
+      const config = payload.context.system.config;
+
+      if (config.dev.demo_auth == false || config.dev.demo_password == null) {
+        throw new micro.errors.AuthorizationError(['Demo auth disabled']);
+      }
+
+      if (password == config.dev.demo_password) {
+        const token = await issueDevToken(payload.request, user, payload.context.system.config);
+        return { token, user_id: user };
+      } else {
+        throw new micro.errors.AuthorizationError(['Authentication failed']);
+      }
+    }
+  });
+
+const TokenParams = t.object({});
+
+export const token: RouteGenerator = (router) =>
+  router.post('/dev/token.json', {
+    validator: micro.schema.createTsCodecValidator(TokenParams, { allowAdditional: true }),
+    authorize: authDevUser,
+    handler: async (payload) => {
+      const { user_id } = payload.context;
+      const outToken = await issuePowerSyncToken(payload.request, user_id!, payload.context.system.config);
+      return { token: outToken, user_id: user_id, endpoint: endpoint(payload.request) };
+    }
+  });
+
+const OpType = {
+  PUT: 'PUT',
+  PATCH: 'PATCH',
+  DELETE: 'DELETE'
+};
+
+const CrudEntry = t.object({
+  op: t.Enum(OpType),
+  type: t.string,
+  id: t.string,
+  op_id: t.number.optional(),
+  data: t.any.optional()
+});
+
+const CrudRequest = t.object({
+  data: t.array(CrudEntry),
+  write_checkpoint: t.boolean.optional()
+});
+export const crud: RouteGenerator = (router) =>
+  router.post('/crud.json', {
+    validator: micro.schema.createTsCodecValidator(CrudRequest, { allowAdditional: true }),
+    authorize: authUser,
+
+    handler: async (payload) => {
+      const { user_id, system } = payload.context;
+
+      const pool = system.requirePgPool();
+
+      if (!system.config.dev.crud_api) {
+        throw new Error('CRUD api disabled');
+      }
+
+      const params = payload.params;
+
+      let statements: pgwire.Statement[] = [];
+
+      // Implementation note:
+      // Postgres does automatic "assigment cast" for query literals,
+      // e.g. a string literal to uuid. However, the same doesn't apply
+      // to query parameters.
+      // To handle those automatically, we use `json_populate_record`
+      // to automatically cast to the correct types.
+
+      for (let op of params.data) {
+        const table = util.escapeIdentifier(op.type);
+        if (op.op == 'PUT') {
+          const data = op.data as Record<string, any>;
+          const with_id = { ...data, id: op.id };
+
+          const columnsEscaped = Object.keys(with_id).map(util.escapeIdentifier);
+          const columnsJoined = columnsEscaped.join(', ');
+
+          let updateClauses: string[] = [];
+
+          for (let key of Object.keys(data)) {
+            updateClauses.push(`${util.escapeIdentifier(key)} = EXCLUDED.${util.escapeIdentifier(key)}`);
+          }
+
+          const updateClause = updateClauses.length > 0 ? `DO UPDATE SET ${updateClauses.join(', ')}` : `DO NOTHING`;
+
+          const statement = `
+          WITH data_row AS (
+              SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          INSERT INTO ${table} (${columnsJoined})
+          SELECT ${columnsJoined} FROM data_row
+          ON CONFLICT(id) ${updateClause}`;
+
+          statements.push({
+            statement: statement,
+            params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
+          });
+        } else if (op.op == 'PATCH') {
+          const data = op.data as Record<string, any>;
+          const with_id = { ...data, id: op.id };
+
+          let updateClauses: string[] = [];
+
+          for (let key of Object.keys(data)) {
+            updateClauses.push(`${util.escapeIdentifier(key)} = data_row.${util.escapeIdentifier(key)}`);
+          }
+
+          const statement = `
+          WITH data_row AS (
+              SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          UPDATE ${table}
+          SET ${updateClauses.join(', ')}
+          FROM data_row
+          WHERE ${table}.id = data_row.id`;
+
+          statements.push({
+            statement: statement,
+            params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
+          });
+        } else if (op.op == 'DELETE') {
+          statements.push({
+            statement: `
+          WITH data_row AS (
+            SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          DELETE FROM ${table}
+          USING data_row
+          WHERE ${table}.id = data_row.id`,
+            params: [{ type: 'varchar', value: JSON.stringify({ id: op.id }) }]
+          });
+        }
+      }
+      await pool.query(...statements);
+
+      const storage = system.storage;
+      if (payload.params.write_checkpoint === true) {
+        const write_checkpoint = await util.createWriteCheckpoint(pool, storage, payload.context.user_id!);
+        return { write_checkpoint: String(write_checkpoint) };
+      } else if (payload.params.write_checkpoint === false) {
+        return {};
+      } else {
+        // Legacy
+        const checkpoint = await util.getClientCheckpoint(pool, storage);
+        return {
+          checkpoint
+        };
+      }
+    }
+  });
+
+export const dev_routes = [auth, auth2, token, crud];

package/src/routes/route-generators.ts

@@ -0,0 +1,39 @@
+import * as micro from '@journeyapps-platform/micro';
+
+import { Context, RouteGenerator } from './router.js';
+import { admin_routes } from './admin.js';
+import { writeCheckpoint, writeCheckpoint2 } from './checkpointing.js';
+import { dev_routes } from './dev.js';
+import { syncRulesRoutes } from './sync-rules.js';
+import { IReactiveStream, ReactiveSocketRouter } from '@powersync/service-rsocket-router';
+import { sync_stream_reactive } from './socket-route.js';
+import { syncStreamed } from './sync-stream.js';
+
+/**
+ * Generates router endpoints using the specified router instance
+ */
+export const generateHTTPRoutes = (router: micro.fastify.FastifyRouter<Context>): micro.router.Route[] => {
+  const generators: RouteGenerator[] = [
+    ...dev_routes,
+    writeCheckpoint,
+    writeCheckpoint2,
+    ...syncRulesRoutes,
+    ...admin_routes
+  ];
+
+  return generators.map((generateRoute) => generateRoute(router));
+};
+
+/**
+ * Generates route endpoint for HTTP sync streaming
+ */
+export const generateHTTPStreamRoutes = (router: micro.fastify.FastifyRouter<Context>): micro.router.Route[] => {
+  return [syncStreamed].map((generateRoute) => generateRoute(router));
+};
+
+/**
+ *  Generates socket router endpoints using the specified router instance
+ */
+export const generateSocketRoutes = (router: ReactiveSocketRouter<Context>): IReactiveStream[] => {
+  return [sync_stream_reactive].map((generateRoute) => generateRoute(router));
+};