@powersync/service-core 0.2.2 → 0.3.0

package/src/replication/WalStreamManager.ts

@@ -1,14 +1,13 @@
 import * as pgwire from '@powersync/service-jpgwire';
-import * as micro from '@journeyapps-platform/micro';
 import { hrtime } from 'node:process';
 
-import * as storage from '@/storage/storage-index.js';
-import * as util from '@/util/util-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 import { DefaultErrorRateLimiter } from './ErrorRateLimiter.js';
-import { touch } from './WalStream.js';
 import { WalStreamRunner } from './WalStreamRunner.js';
 import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { container, logger } from '@powersync/lib-services-framework';
 
 // 5 minutes
 const PING_INTERVAL = 1_000_000_000n * 300n;
@@ -37,8 +36,8 @@ export class WalStreamManager {
 
   start() {
     this.runLoop().catch((e) => {
-      micro.logger.error(`Fatal WalStream error`, e);
-      util.captureException(e);
+      logger.error(`Fatal WalStream error`, e);
+      container.reporter.captureException(e);
       setTimeout(() => {
         process.exit(1);
       }, 1000);
@@ -58,7 +57,7 @@ export class WalStreamManager {
     const configured_sync_rules = await util.loadSyncRules(this.system.config);
     let configured_lock: storage.ReplicationLock | undefined = undefined;
     if (configured_sync_rules != null) {
-      micro.logger.info('Loading sync rules from configuration');
+      logger.info('Loading sync rules from configuration');
       try {
         // Configure new sync rules, if it has changed.
         // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
@@ -70,13 +69,13 @@ export class WalStreamManager {
         }
       } catch (e) {
         // Log, but continue with previous sync rules
-        micro.logger.error(`Failed to load sync rules from configuration`, e);
+        logger.error(`Failed to load sync rules from configuration`, e);
       }
     } else {
-      micro.logger.info('No sync rules configured - configure via API');
+      logger.info('No sync rules configured - configure via API');
     }
     while (!this.stopped) {
-      await touch();
+      await container.probes.touch();
       try {
         const pool = this.system.pgwire_pool;
         if (pool) {
@@ -93,7 +92,7 @@ export class WalStreamManager {
           }
         }
       } catch (e) {
-        micro.logger.error(`Failed to refresh wal streams`, e);
+        logger.error(`Failed to refresh wal streams`, e);
       }
       await new Promise((resolve) => setTimeout(resolve, 5000));
     }
@@ -117,7 +116,7 @@ export class WalStreamManager {
       try {
         await db.query(`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`);
       } catch (e) {
-        micro.logger.warn(`Failed to ping`, e);
+        logger.warn(`Failed to ping`, e);
       }
       this.lastPing = now;
     }
@@ -168,7 +167,7 @@ export class WalStreamManager {
           // for example from stricter validation that was added.
           // This will be retried every couple of seconds.
           // When new (valid) sync rules are deployed and processed, this one be disabled.
-          micro.logger.error(`Failed to start replication for ${syncRules.slot_name}`, e);
+          logger.error(`Failed to start replication for ${syncRules.slot_name}`, e);
         }
       }
     }
@@ -184,7 +183,7 @@ export class WalStreamManager {
         await stream.terminate();
       } catch (e) {
         // This will be retried
-        micro.logger.warn(`Failed to terminate ${stream.slot_name}`, e);
+        logger.warn(`Failed to terminate ${stream.slot_name}`, e);
       }
     }
 
@@ -207,7 +206,7 @@ export class WalStreamManager {
           await lock.release();
         }
       } catch (e) {
-        micro.logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
+        logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
       }
     }
   }

package/src/replication/WalStreamRunner.ts

@@ -1,12 +1,12 @@
 import * as pgwire from '@powersync/service-jpgwire';
-import * as micro from '@journeyapps-platform/micro';
 
-import * as storage from '@/storage/storage-index.js';
-import * as util from '@/util/util-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 import { ErrorRateLimiter } from './ErrorRateLimiter.js';
 import { MissingReplicationSlotError, WalStream } from './WalStream.js';
 import { ResolvedConnection } from '../util/config/types.js';
+import { container, logger } from '@powersync/lib-services-framework';
 
 export interface WalStreamRunnerOptions {
   factory: storage.BucketStorageFactory;
@@ -46,12 +46,12 @@ export class WalStreamRunner {
       await this.replicateLoop();
     } catch (e) {
       // Fatal exception
-      util.captureException(e, {
+      container.reporter.captureException(e, {
         metadata: {
           replication_slot: this.slot_name
         }
       });
-      micro.logger.error(`Replication failed on ${this.slot_name}`, e);
+      logger.error(`Replication failed on ${this.slot_name}`, e);
 
       if (e instanceof MissingReplicationSlotError) {
         // This stops replication on this slot, and creates a new slot
@@ -96,7 +96,7 @@ export class WalStreamRunner {
       });
       await stream.replicate();
     } catch (e) {
-      micro.logger.error(`Replication error`, e);
+      logger.error(`Replication error`, e);
       if (e.cause != null) {
         // Example:
         // PgError.conn_ended: Unable to do postgres query on ended connection
@@ -118,13 +118,13 @@ export class WalStreamRunner {
         //   [Symbol(pg.ErrorResponse)]: undefined
         // }
         // Without this additional log, the cause would not be visible in the logs.
-        micro.logger.error(`cause`, e.cause);
+        logger.error(`cause`, e.cause);
       }
       if (e instanceof MissingReplicationSlotError) {
         throw e;
       } else {
         // Report the error if relevant, before retrying
-        util.captureException(e, {
+        container.reporter.captureException(e, {
           metadata: {
             replication_slot: this.slot_name
           }
@@ -144,7 +144,7 @@ export class WalStreamRunner {
    * This will also release the lock if start() was called earlier.
    */
   async stop(options?: { force?: boolean }) {
-    micro.logger.info(`${this.slot_name} Stopping replication`);
+    logger.info(`${this.slot_name} Stopping replication`);
     // End gracefully
     this.abortController.abort();
 
@@ -161,7 +161,7 @@ export class WalStreamRunner {
    * Stops replication if needed.
    */
   async terminate(options?: { force?: boolean }) {
-    micro.logger.info(`${this.slot_name} Terminating replication`);
+    logger.info(`${this.slot_name} Terminating replication`);
     await this.stop(options);
 
     const slotName = this.slot_name;

package/src/replication/util.ts

@@ -1,6 +1,6 @@
 import * as pgwire from '@powersync/service-jpgwire';
 
-import * as util from '@/util/util-index.js';
+import * as util from '../util/util-index.js';
 import { ReplicationColumn, ReplicationIdentity } from './PgRelation.js';
 
 export interface ReplicaIdentityResult {

package/src/routes/auth.ts

@@ -1,19 +1,17 @@
-import * as micro from '@journeyapps-platform/micro';
-import { FastifyRequest } from 'fastify';
 import * as jose from 'jose';
 
-import * as auth from '@/auth/auth-index.js';
-import * as util from '@/util/util-index.js';
-import { Context } from './router.js';
+import * as auth from '../auth/auth-index.js';
+import * as util from '../util/util-index.js';
+import { BasicRouterRequest, Context, RequestEndpointHandlerPayload } from './router.js';
 import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
 
-export function endpoint(req: FastifyRequest) {
+export function endpoint(req: BasicRouterRequest) {
   const protocol = req.headers['x-forwarded-proto'] ?? req.protocol;
   const host = req.hostname;
   return `${protocol}://${host}`;
 }
 
-function devAudience(req: FastifyRequest): string {
+function devAudience(req: BasicRouterRequest): string {
   return `${endpoint(req)}/dev`;
 }
 
@@ -22,7 +20,7 @@ function devAudience(req: FastifyRequest): string {
  *
  * Will be replaced by temporary tokens issued by PowerSync Management service.
  */
-export async function issueDevToken(req: FastifyRequest, user_id: string, config: util.ResolvedPowerSyncConfig) {
+export async function issueDevToken(req: BasicRouterRequest, user_id: string, config: util.ResolvedPowerSyncConfig) {
   const iss = devAudience(req);
   const aud = devAudience(req);
 
@@ -42,7 +40,11 @@ export async function issueDevToken(req: FastifyRequest, user_id: string, config
 }
 
 /** @deprecated */
-export async function issueLegacyDevToken(req: FastifyRequest, user_id: string, config: util.ResolvedPowerSyncConfig) {
+export async function issueLegacyDevToken(
+  req: BasicRouterRequest,
+  user_id: string,
+  config: util.ResolvedPowerSyncConfig
+) {
   const iss = devAudience(req);
   const aud = config.jwt_audiences[0];
 
@@ -61,7 +63,11 @@ export async function issueLegacyDevToken(req: FastifyRequest, user_id: string,
     .sign(key.key);
 }
 
-export async function issuePowerSyncToken(req: FastifyRequest, user_id: string, config: util.ResolvedPowerSyncConfig) {
+export async function issuePowerSyncToken(
+  req: BasicRouterRequest,
+  user_id: string,
+  config: util.ResolvedPowerSyncConfig
+) {
   const iss = devAudience(req);
   const aud = config.jwt_audiences[0];
   const key = config.dev.dev_key;
@@ -89,8 +95,8 @@ export function getTokenFromHeader(authHeader: string = ''): string | null {
   return token ?? null;
 }
 
-export const authUser = async (payload: micro.fastify.FastifyHandlerPayload<any, Context>) => {
-  return authorizeUser(payload.context, payload.request.headers.authorization);
+export const authUser = async (payload: RequestEndpointHandlerPayload) => {
+  return authorizeUser(payload.context, payload.request.headers.authorization as string);
 };
 
 export async function authorizeUser(context: Context, authHeader: string = '') {
@@ -142,9 +148,9 @@ export async function generateContext(system: CorePowerSyncSystem, token: string
 /**
  * @deprecated
  */
-export const authDevUser = async (payload: micro.fastify.FastifyHandlerPayload<any, Context>) => {
+export const authDevUser = async (payload: RequestEndpointHandlerPayload) => {
   const context = payload.context;
-  const token = getTokenFromHeader(payload.request.headers.authorization);
+  const token = getTokenFromHeader(payload.request.headers.authorization as string);
   if (!context.system.config.dev.demo_auth) {
     return {
       authorized: false,
@@ -179,7 +185,7 @@ export const authDevUser = async (payload: micro.fastify.FastifyHandlerPayload<a
   return { authorized: true };
 };
 
-export const authApi = (payload: micro.fastify.FastifyHandlerPayload<any, Context>) => {
+export const authApi = (payload: RequestEndpointHandlerPayload) => {
   const context = payload.context;
   const api_keys = context.system.config.api_tokens;
   if (api_keys.length == 0) {
@@ -188,7 +194,7 @@ export const authApi = (payload: micro.fastify.FastifyHandlerPayload<any, Contex
       errors: ['Authentication disabled']
     };
   }
-  const auth = payload.request.headers.authorization ?? '';
+  const auth = (payload.request.headers.authorization as string) ?? '';
 
   const tokenMatch = /^(Token|Bearer) (\S+)$/.exec(auth);
   if (!tokenMatch) {

package/src/routes/endpoints/admin.ts

@@ -0,0 +1,237 @@
+import { errors, router, schema } from '@powersync/lib-services-framework';
+import { SqlSyncRules, SqliteValue, StaticSchema, isJsonValue, toSyncRulesValue } from '@powersync/service-sync-rules';
+import { internal_routes } from '@powersync/service-types';
+
+import * as api from '../../api/api-index.js';
+import * as util from '../../util/util-index.js';
+
+import { routeDefinition } from '../router.js';
+import { PersistedSyncRulesContent } from '../../storage/BucketStorage.js';
+import { authApi } from '../auth.js';
+
+const demoCredentials = routeDefinition({
+  path: '/api/admin/v1/demo-credentials',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.DemoCredentialsRequest, {
+    allowAdditional: true
+  }),
+  handler: async (payload) => {
+    const connection = payload.context.system.config.connection;
+    if (connection == null || !connection.demo_database) {
+      return internal_routes.DemoCredentialsResponse.encode({});
+    }
+
+    const uri = util.buildDemoPgUri(connection);
+    return internal_routes.DemoCredentialsResponse.encode({
+      credentials: {
+        postgres_uri: uri
+      }
+    });
+  }
+});
+
+export const executeSql = routeDefinition({
+  path: '/api/admin/v1/execute-sql',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.ExecuteSqlRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const connection = payload.context.system.config.connection;
+    if (connection == null || !connection.debug_api) {
+      return internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: 'SQL querying is not enabled'
+      });
+    }
+
+    const pool = payload.context.system.requirePgPool();
+
+    const { query, args } = payload.params.sql;
+
+    try {
+      const result = await pool.query({
+        statement: query,
+        params: args.map(util.autoParameter)
+      });
+
+      return internal_routes.ExecuteSqlResponse.encode({
+        success: true,
+        results: {
+          columns: result.columns.map((c) => c.name),
+          rows: result.rows.map((row) => {
+            return row.map((value) => mapColumnValue(toSyncRulesValue(value)));
+          })
+        }
+      });
+    } catch (e) {
+      return internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: e.message
+      });
+    }
+  }
+});
+
+export const diagnostics = routeDefinition({
+  path: '/api/admin/v1/diagnostics',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.DiagnosticsRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const include_content = payload.params.sync_rules_content ?? false;
+    const system = payload.context.system;
+
+    const status = await api.getConnectionStatus(system);
+    if (status == null) {
+      return internal_routes.DiagnosticsResponse.encode({
+        connections: []
+      });
+    }
+
+    const { storage } = system;
+    const active = await storage.getActiveSyncRulesContent();
+    const next = await storage.getNextSyncRulesContent();
+
+    const active_status = await api.getSyncRulesStatus(active, system, {
+      include_content,
+      check_connection: status.connected,
+      live_status: true
+    });
+
+    const next_status = await api.getSyncRulesStatus(next, system, {
+      include_content,
+      check_connection: status.connected,
+      live_status: true
+    });
+
+    return internal_routes.DiagnosticsResponse.encode({
+      connections: [status],
+      active_sync_rules: active_status,
+      deploying_sync_rules: next_status
+    });
+  }
+});
+
+export const getSchema = routeDefinition({
+  path: '/api/admin/v1/schema',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.GetSchemaRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(system));
+  }
+});
+
+export const reprocess = routeDefinition({
+  path: '/api/admin/v1/reprocess',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.ReprocessRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    const storage = system.storage;
+    const next = await storage.getNextSyncRules();
+    if (next != null) {
+      throw new Error(`Busy processing sync rules - cannot reprocess`);
+    }
+
+    const active = await storage.getActiveSyncRules();
+    if (active == null) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'NO_SYNC_RULES',
+        description: 'No active sync rules'
+      });
+    }
+
+    const new_rules = await storage.updateSyncRules({
+      content: active.sync_rules.content
+    });
+
+    return internal_routes.ReprocessResponse.encode({
+      connections: [
+        {
+          tag: system.config.connection!.tag,
+          id: system.config.connection!.id,
+          slot_name: new_rules.slot_name
+        }
+      ]
+    });
+  }
+});
+
+export const validate = routeDefinition({
+  path: '/api/admin/v1/validate',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.ValidateRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    const content = payload.params.sync_rules;
+
+    const schemaData = await api.getConnectionsSchema(system);
+    const schema = new StaticSchema(schemaData.connections);
+
+    const sync_rules: PersistedSyncRulesContent = {
+      // Dummy values
+      id: 0,
+      slot_name: '',
+
+      parsed() {
+        return {
+          ...this,
+          sync_rules: SqlSyncRules.fromYaml(content, { throwOnError: false, schema })
+        };
+      },
+      sync_rules_content: content,
+      async lock() {
+        throw new Error('Lock not implemented');
+      }
+    };
+
+    const connectionStatus = await api.getConnectionStatus(system);
+    if (connectionStatus == null) {
+      return internal_routes.ValidateResponse.encode({
+        errors: [{ level: 'fatal', message: 'No connection configured' }],
+        connections: []
+      });
+    }
+
+    const status = (await api.getSyncRulesStatus(sync_rules, system, {
+      include_content: false,
+      check_connection: connectionStatus?.connected,
+      live_status: false
+    }))!;
+
+    if (connectionStatus == null) {
+      status.errors.push({ level: 'fatal', message: 'No connection configured' });
+    }
+
+    return internal_routes.ValidateResponse.encode(status);
+  }
+});
+
+function mapColumnValue(value: SqliteValue) {
+  if (typeof value == 'bigint') {
+    return Number(value);
+  } else if (isJsonValue(value)) {
+    return value;
+  } else {
+    return null;
+  }
+}
+
+export const ADMIN_ROUTES = [demoCredentials, executeSql, diagnostics, getSchema, reprocess, validate];

package/src/routes/endpoints/checkpointing.ts

@@ -0,0 +1,41 @@
+import * as t from 'ts-codec';
+import { router, schema } from '@powersync/lib-services-framework';
+
+import * as util from '../../util/util-index.js';
+import { authUser } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+const WriteCheckpointRequest = t.object({});
+
+export const writeCheckpoint = routeDefinition({
+  path: '/write-checkpoint.json',
+  method: router.HTTPMethod.GET,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+    const storage = system.storage;
+
+    const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
+    return {
+      checkpoint
+    };
+  }
+});
+
+export const writeCheckpoint2 = routeDefinition({
+  path: '/write-checkpoint2.json',
+  method: router.HTTPMethod.GET,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const { user_id, system } = payload.context;
+    const storage = system.storage;
+    const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, user_id!);
+    return {
+      write_checkpoint: String(write_checkpoint)
+    };
+  }
+});
+
+export const CHECKPOINT_ROUTES = [writeCheckpoint, writeCheckpoint2];