@powersync/service-core 0.8.7 → 0.9.0

package/src/routes/endpoints/admin.ts

@@ -1,11 +1,9 @@
 import { errors, router, schema } from '@powersync/lib-services-framework';
-import { SqlSyncRules, SqliteValue, StaticSchema, isJsonValue, toSyncRulesValue } from '@powersync/service-sync-rules';
+import { SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
 import { internal_routes } from '@powersync/service-types';
 
 import * as api from '../../api/api-index.js';
-import * as util from '../../util/util-index.js';
-
-import { PersistedSyncRulesContent } from '../../storage/BucketStorage.js';
+import * as storage from '../../storage/storage-index.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
@@ -15,47 +13,27 @@ export const executeSql = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ExecuteSqlRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const connection = payload.context.system.config.connection;
-    if (connection == null || !connection.debug_api) {
-      return internal_routes.ExecuteSqlResponse.encode({
-        results: {
-          columns: [],
-          rows: []
-        },
-        success: false,
-        error: 'SQL querying is not enabled'
-      });
-    }
-
-    const pool = payload.context.system.requirePgPool();
-
-    const { query, args } = payload.params.sql;
+    const {
+      params: {
+        sql: { query, args }
+      }
+    } = payload;
 
-    try {
-      const result = await pool.query({
-        statement: query,
-        params: args.map(util.autoParameter)
-      });
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-      return internal_routes.ExecuteSqlResponse.encode({
-        success: true,
-        results: {
-          columns: result.columns.map((c) => c.name),
-          rows: result.rows.map((row) => {
-            return row.map((value) => mapColumnValue(toSyncRulesValue(value)));
-          })
-        }
-      });
-    } catch (e) {
+    const sourceConfig = await apiHandler.getSourceConfig();
+    if (!sourceConfig.debug_api) {
       return internal_routes.ExecuteSqlResponse.encode({
         results: {
           columns: [],
           rows: []
         },
         success: false,
-        error: e.message
+        error: 'SQL querying is not enabled'
       });
     }
+
+    return internal_routes.ExecuteSqlResponse.encode(await apiHandler.executeQuery(query, args));
   }
 });
 
@@ -65,34 +43,45 @@ export const diagnostics = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.DiagnosticsRequest, { allowAdditional: true }),
   handler: async (payload) => {
+    const { context } = payload;
+    const { service_context } = context;
     const include_content = payload.params.sync_rules_content ?? false;
-    const system = payload.context.system;
 
-    const status = await api.getConnectionStatus(system);
-    if (status == null) {
+    const apiHandler = service_context.routerEngine!.getAPI();
+
+    const status = await apiHandler.getConnectionStatus();
+    if (!status) {
       return internal_routes.DiagnosticsResponse.encode({
         connections: []
       });
     }
 
-    const { storage } = system;
-    const active = await storage.getActiveSyncRulesContent();
-    const next = await storage.getNextSyncRulesContent();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const active = await activeBucketStorage.getActiveSyncRulesContent();
+    const next = await activeBucketStorage.getNextSyncRulesContent();
 
-    const active_status = await api.getSyncRulesStatus(active, system, {
+    const active_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, active, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
-    const next_status = await api.getSyncRulesStatus(next, system, {
+    const next_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, next, {
       include_content,
       check_connection: status.connected,
       live_status: true
     });
 
     return internal_routes.DiagnosticsResponse.encode({
-      connections: [status],
+      connections: [
+        {
+          ...status,
+          // TODO update this in future
+          postgres_uri: status.uri
+        }
+      ],
       active_sync_rules: active_status,
       deploying_sync_rules: next_status
     });
@@ -105,9 +94,9 @@ export const getSchema = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.GetSchemaRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
 
-    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(system));
+    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(apiHandler));
   }
 });
 
@@ -117,15 +106,19 @@ export const reprocess = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ReprocessRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
-    const storage = system.storage;
-    const next = await storage.getNextSyncRules();
+    const {
+      context: { service_context }
+    } = payload;
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+    const apiHandler = service_context.routerEngine!.getAPI();
+    const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
     if (next != null) {
       throw new Error(`Busy processing sync rules - cannot reprocess`);
     }
 
-    const active = await storage.getActiveSyncRules();
+    const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
     if (active == null) {
       throw new errors.JourneyError({
         status: 422,
@@ -134,15 +127,18 @@ export const reprocess = routeDefinition({
       });
     }
 
-    const new_rules = await storage.updateSyncRules({
+    const new_rules = await activeBucketStorage.updateSyncRules({
       content: active.sync_rules.content
     });
 
+    const baseConfig = await apiHandler.getSourceConfig();
+
     return internal_routes.ReprocessResponse.encode({
       connections: [
         {
-          tag: system.config.connection!.tag,
-          id: system.config.connection!.id,
+          // Previously the connection was asserted with `!`
+          tag: baseConfig.tag,
+          id: baseConfig.id,
           slot_name: new_rules.slot_name
         }
       ]
@@ -156,14 +152,16 @@ export const validate = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(internal_routes.ValidateRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-
+    const {
+      context: { service_context }
+    } = payload;
     const content = payload.params.sync_rules;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const schemaData = await api.getConnectionsSchema(system);
+    const schemaData = await api.getConnectionsSchema(apiHandler);
     const schema = new StaticSchema(schemaData.connections);
 
-    const sync_rules: PersistedSyncRulesContent = {
+    const sync_rules: storage.PersistedSyncRulesContent = {
       // Dummy values
       id: 0,
       slot_name: '',
@@ -171,7 +169,10 @@ export const validate = routeDefinition({
       parsed() {
         return {
           ...this,
-          sync_rules: SqlSyncRules.fromYaml(content, { throwOnError: false, schema })
+          sync_rules: SqlSyncRules.fromYaml(content, {
+            ...apiHandler.getParseSyncRulesOptions(),
+            schema
+          })
         };
       },
       sync_rules_content: content,
@@ -180,19 +181,24 @@ export const validate = routeDefinition({
       }
     };
 
-    const connectionStatus = await api.getConnectionStatus(system);
-    if (connectionStatus == null) {
+    const connectionStatus = await apiHandler.getConnectionStatus();
+    if (!connectionStatus) {
       return internal_routes.ValidateResponse.encode({
         errors: [{ level: 'fatal', message: 'No connection configured' }],
         connections: []
       });
     }
 
-    const status = (await api.getSyncRulesStatus(sync_rules, system, {
-      include_content: false,
-      check_connection: connectionStatus?.connected,
-      live_status: false
-    }))!;
+    const status = (await api.getSyncRulesStatus(
+      service_context.storageEngine.activeBucketStorage,
+      apiHandler,
+      sync_rules,
+      {
+        include_content: false,
+        check_connection: connectionStatus.connected,
+        live_status: false
+      }
+    ))!;
 
     if (connectionStatus == null) {
       status.errors.push({ level: 'fatal', message: 'No connection configured' });
@@ -202,14 +208,4 @@ export const validate = routeDefinition({
   }
 });
 
-function mapColumnValue(value: SqliteValue) {
-  if (typeof value == 'bigint') {
-    return Number(value);
-  } else if (isJsonValue(value)) {
-    return value;
-  } else {
-    return null;
-  }
-}
-
 export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate];

package/src/routes/endpoints/checkpointing.ts

@@ -1,6 +1,7 @@
+import { logger, router, schema } from '@powersync/lib-services-framework';
 import * as t from 'ts-codec';
-import { router, schema } from '@powersync/lib-services-framework';
 
+import * as framework from '@powersync/lib-services-framework';
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
@@ -15,13 +16,33 @@ export const writeCheckpoint = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
-    const storage = system.storage;
+    const {
+      context: { service_context }
+    } = payload;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
-    return {
-      checkpoint
-    };
+    // This old API needs a persisted checkpoint id.
+    // Since we don't use LSNs anymore, the only way to get that is to wait.
+    const start = Date.now();
+
+    const head = await apiHandler.getReplicationHead();
+
+    const timeout = 50_000;
+
+    logger.info(`Waiting for LSN checkpoint: ${head}`);
+    while (Date.now() - start < timeout) {
+      const cp = await service_context.storageEngine.activeBucketStorage.getActiveCheckpoint();
+      if (!cp.hasSyncRules()) {
+        throw new Error('No sync rules available');
+      }
+      if (cp.lsn && cp.lsn >= head) {
+        logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);
+        return { checkpoint: cp.checkpoint };
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 30));
+    }
+    throw new Error('Timeout while waiting for checkpoint');
   }
 });
 
@@ -31,13 +52,32 @@ export const writeCheckpoint2 = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const { user_id, system } = payload.context;
+    const { user_id, service_context } = payload.context;
+
+    const apiHandler = service_context.routerEngine!.getAPI();
+
     const client_id = payload.params.client_id;
     const full_user_id = util.checkpointUserId(user_id, client_id);
-    const storage = system.storage;
-    const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, full_user_id);
+
+    const currentCheckpoint = await apiHandler.getReplicationHead();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+
+    const activeSyncRules = await activeBucketStorage.getActiveSyncRulesContent();
+    if (!activeSyncRules) {
+      throw new framework.errors.ValidationError(`Cannot create Write Checkpoint since no sync rules are active.`);
+    }
+
+    using syncBucketStorage = activeBucketStorage.getInstance(activeSyncRules);
+    const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
+      user_id: full_user_id,
+      heads: { '1': currentCheckpoint }
+    });
+    logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`);
+
     return {
-      write_checkpoint: String(write_checkpoint)
+      write_checkpoint: String(writeCheckpoint)
     };
   }
 });

package/src/routes/endpoints/probes.ts

@@ -0,0 +1,58 @@
+import { container, router } from "@powersync/lib-services-framework";
+import { routeDefinition } from "../router.js";
+
+export enum ProbeRoutes {
+  STARTUP = '/probes/startup',
+  LIVENESS = '/probes/liveness',
+  READINESS = '/probes/readiness'
+}
+
+export const startupCheck = routeDefinition({
+  path: ProbeRoutes.STARTUP,
+  method: router.HTTPMethod.GET,
+  handler: async () => {
+    const state = container.probes.state();
+
+    return new router.RouterResponse({
+      status: state.started ? 200 : 400,
+      data: {
+        ...state,
+      }
+    });
+  }
+});
+
+export const livenessCheck = routeDefinition({
+  path: ProbeRoutes.LIVENESS,
+  method: router.HTTPMethod.GET,
+  handler: async () => {
+    const state = container.probes.state();
+
+    const timeDifference = Date.now() - state.touched_at.getTime()
+    const status = timeDifference < 10000 ? 200 : 400;
+
+    return new router.RouterResponse({
+      status,
+      data: {
+        ...state,
+      }
+    });
+  }
+});
+
+export const readinessCheck = routeDefinition({
+  path: ProbeRoutes.READINESS,
+  method: router.HTTPMethod.GET,
+  handler: async () => {
+    const state = container.probes.state();
+
+    return new router.RouterResponse({
+      status: state.ready ? 200 : 400,
+      data: {
+        ...state,
+      }
+    });
+  }
+});
+
+export const PROBES_ROUTES = [startupCheck, livenessCheck, readinessCheck];

package/src/routes/endpoints/socket-route.ts

@@ -12,7 +12,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
     handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
-      const { system } = context;
+      const { service_context } = context;
+      const { routerEngine } = service_context;
 
       // Create our own controller that we can abort directly
       const controller = new AbortController();
@@ -31,7 +32,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         }
       });
 
-      if (system.closed) {
+      if (routerEngine!.closed) {
         responder.onError(
           new errors.JourneyError({
             status: 503,
@@ -45,9 +46,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
 
       const syncParams = new RequestParameters(context.token_payload!, params.parameters ?? {});
 
-      const storage = system.storage;
+      const {
+        storageEngine: { activeBucketStorage }
+      } = service_context;
       // Sanity check before we start the stream
-      const cp = await storage.getActiveCheckpoint();
+      const cp = await activeBucketStorage.getActiveCheckpoint();
       if (!cp.hasSyncRules()) {
         responder.onError(
           new errors.JourneyError({
@@ -60,7 +63,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
-      const removeStopHandler = system.addStopHandler(() => {
+      const removeStopHandler = routerEngine!.addStopHandler(() => {
         controller.abort();
       });
 
@@ -68,7 +71,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       const tracker = new sync.RequestTracker();
       try {
         for await (const data of sync.streamResponse({
-          storage,
+          storage: activeBucketStorage,
+          parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
           params: {
             ...params,
             binary_data: true // always true for web sockets

package/src/routes/endpoints/sync-rules.ts

@@ -1,10 +1,9 @@
-import * as t from 'ts-codec';
-import type { FastifyPluginAsync } from 'fastify';
-import * as pgwire from '@powersync/service-jpgwire';
 import { errors, router, schema } from '@powersync/lib-services-framework';
 import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
+import type { FastifyPluginAsync } from 'fastify';
+import * as t from 'ts-codec';
 
-import * as replication from '../../replication/replication-index.js';
+import { RouteAPI } from '../../api/RouteAPI.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
@@ -39,7 +38,10 @@ export const deploySyncRules = routeDefinition({
   plugins: [yamlPlugin],
   validator: schema.createTsCodecValidator(DeploySyncRulesRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    if (payload.context.system.config.sync_rules.present) {
+    const { service_context } = payload.context;
+    const { storageEngine } = service_context;
+
+    if (service_context.configuration.sync_rules.present) {
       // If sync rules are configured via the config, disable deploy via the API.
       throw new errors.JourneyError({
         status: 422,
@@ -51,7 +53,12 @@ export const deploySyncRules = routeDefinition({
     const content = payload.params.content;
 
     try {
-      SqlSyncRules.fromYaml(payload.params.content);
+      const apiHandler = service_context.routerEngine!.getAPI();
+      SqlSyncRules.fromYaml(payload.params.content, {
+        ...apiHandler.getParseSyncRulesOptions(),
+        // We don't do any schema-level validation at this point
+        schema: undefined
+      });
     } catch (e) {
       throw new errors.JourneyError({
         status: 422,
@@ -61,7 +68,7 @@ export const deploySyncRules = routeDefinition({
       });
     }
 
-    const sync_rules = await payload.context.system.storage.updateSyncRules({
+    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({
       content: content
     });
 
@@ -84,8 +91,10 @@ export const validateSyncRules = routeDefinition({
   validator: schema.createTsCodecValidator(ValidateSyncRulesRequest, { allowAdditional: true }),
   handler: async (payload) => {
     const content = payload.params.content;
+    const { service_context } = payload.context;
+    const apiHandler = service_context.routerEngine!.getAPI();
 
-    const info = await debugSyncRules(payload.context.system.requirePgPool(), content);
+    const info = await debugSyncRules(apiHandler, content);
 
     return replyPrettyJson(info);
   }
@@ -96,8 +105,12 @@ export const currentSyncRules = routeDefinition({
   method: router.HTTPMethod.GET,
   authorize: authApi,
   handler: async (payload) => {
-    const storage = payload.context.system.storage;
-    const sync_rules = await storage.getActiveSyncRulesContent();
+    const { service_context } = payload.context;
+    const {
+      storageEngine: { activeBucketStorage }
+    } = service_context;
+
+    const sync_rules = await activeBucketStorage.getActiveSyncRulesContent();
     if (!sync_rules) {
       throw new errors.JourneyError({
         status: 422,
@@ -105,12 +118,12 @@ export const currentSyncRules = routeDefinition({
         description: 'No active sync rules'
       });
     }
-    const info = await debugSyncRules(payload.context.system.requirePgPool(), sync_rules.sync_rules_content);
-    const next = await storage.getNextSyncRulesContent();
 
-    const next_info = next
-      ? await debugSyncRules(payload.context.system.requirePgPool(), next.sync_rules_content)
-      : null;
+    const apiHandler = service_context.routerEngine!.getAPI();
+    const info = await debugSyncRules(apiHandler, sync_rules.sync_rules_content);
+    const next = await activeBucketStorage.getNextSyncRulesContent();
+
+    const next_info = next ? await debugSyncRules(apiHandler, next.sync_rules_content) : null;
 
     const response = {
       current: {
@@ -140,8 +153,11 @@ export const reprocessSyncRules = routeDefinition({
   authorize: authApi,
   validator: schema.createTsCodecValidator(ReprocessSyncRulesRequest),
   handler: async (payload) => {
-    const storage = payload.context.system.storage;
-    const sync_rules = await storage.getActiveSyncRules();
+    const {
+      storageEngine: { activeBucketStorage }
+    } = payload.context.service_context;
+    const apiHandler = payload.context.service_context.routerEngine!.getAPI();
+    const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
     if (sync_rules == null) {
       throw new errors.JourneyError({
         status: 422,
@@ -150,7 +166,7 @@ export const reprocessSyncRules = routeDefinition({
       });
     }
 
-    const new_rules = await storage.updateSyncRules({
+    const new_rules = await activeBucketStorage.updateSyncRules({
       content: sync_rules.sync_rules.content
     });
     return {
@@ -169,15 +185,15 @@ function replyPrettyJson(payload: any) {
   });
 }
 
-async function debugSyncRules(db: pgwire.PgClient, sync_rules: string) {
+async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
   try {
-    const rules = SqlSyncRules.fromYaml(sync_rules);
-    const source_table_patterns = rules.getSourceTables();
-    const wc = new replication.WalConnection({
-      db: db,
-      sync_rules: rules
+    const rules = SqlSyncRules.fromYaml(sync_rules, {
+      ...apiHandler.getParseSyncRulesOptions(),
+      // No schema-based validation at this point
+      schema: undefined
     });
-    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+    const source_table_patterns = rules.getSourceTables();
+    const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
 
     return {
       valid: true,

package/src/routes/endpoints/sync-stream.ts

@@ -8,7 +8,6 @@ import * as util from '../../util/util-index.js';
 import { Metrics } from '../../metrics/Metrics.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
-import { RequestTracker } from '../../sync/RequestTracker.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -20,12 +19,13 @@ export const syncStreamed = routeDefinition({
   authorize: authUser,
   validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
   handler: async (payload) => {
-    const system = payload.context.system;
+    const { service_context } = payload.context;
+    const { routerEngine, storageEngine } = service_context;
     const headers = payload.request.headers;
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
     const clientId = payload.params.client_id;
 
-    if (system.closed) {
+    if (routerEngine!.closed) {
       throw new errors.JourneyError({
         status: 503,
         code: 'SERVICE_UNAVAILABLE',
@@ -36,9 +36,8 @@ export const syncStreamed = routeDefinition({
     const params: util.StreamingSyncRequest = payload.params;
     const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
 
-    const storage = system.storage;
     // Sanity check before we start the stream
-    const cp = await storage.getActiveCheckpoint();
+    const cp = await storageEngine.activeBucketStorage.getActiveCheckpoint();
     if (!cp.hasSyncRules()) {
       throw new errors.JourneyError({
         status: 500,
@@ -47,14 +46,15 @@ export const syncStreamed = routeDefinition({
       });
     }
     const controller = new AbortController();
-    const tracker = new RequestTracker();
+    const tracker = new sync.RequestTracker();
     try {
       Metrics.getInstance().concurrent_connections.add(1);
       const stream = Readable.from(
         sync.transformToBytesTracked(
           sync.ndjson(
             sync.streamResponse({
-              storage,
+              storage: storageEngine.activeBucketStorage,
+              parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
               params,
               syncParams,
               token: payload.context.token_payload!,
@@ -67,7 +67,7 @@ export const syncStreamed = routeDefinition({
         { objectMode: false, highWaterMark: 16 * 1024 }
       );
 
-      const deregister = system.addStopHandler(() => {
+      const deregister = routerEngine!.addStopHandler(() => {
         // This error is not currently propagated to the client
         controller.abort();
         stream.destroy(new Error('Shutting down system'));