@powersync/service-core 0.0.2

package/src/routes/router-socket.ts

@@ -0,0 +1,13 @@
+import * as t from 'ts-codec';
+import { ReactiveSocketRouter, IReactiveStream } from '@powersync/service-rsocket-router';
+
+import { Context } from './router.js';
+
+export const RSocketContextMeta = t.object({
+  token: t.string
+});
+
+/**
+ * Creates a socket route handler given a router instance
+ */
+export type SocketRouteGenerator = (router: ReactiveSocketRouter<Context>) => IReactiveStream;

package/src/routes/router.ts

@@ -0,0 +1,17 @@
+import * as micro from '@journeyapps-platform/micro';
+
+import * as auth from '@/auth/auth-index.js';
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+
+export type Context = {
+  user_id?: string;
+  system: CorePowerSyncSystem;
+
+  token_payload?: auth.JwtPayload;
+};
+
+/**
+ * Creates a route handler given a router instance
+ * TODO move away from Fastify specific types
+ */
+export type RouteGenerator = (router: micro.fastify.FastifyRouter<Context>) => micro.router.Route;

package/src/routes/routes-index.ts

@@ -0,0 +1,5 @@
+export * from './router.js';
+export * from './router-socket.js';
+export * from './route-generators.js';
+
+export * as auth from './auth.js';

package/src/routes/socket-route.ts

@@ -0,0 +1,131 @@
+import { serialize } from 'bson';
+import { SyncParameters, normalizeTokenParameters } from '@powersync/service-sync-rules';
+import * as micro from '@journeyapps-platform/micro';
+
+import * as util from '@/util/util-index.js';
+import { concurrent_connections, data_synced_bytes } from '../metrics/metrics.js';
+import { streamResponse } from '../sync/sync.js';
+import { SyncRoutes } from './sync-stream.js';
+import { SocketRouteGenerator } from './router-socket.js';
+
+export const sync_stream_reactive: SocketRouteGenerator = (router) =>
+  router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
+    authorize: ({ context }) => {
+      return {
+        authorized: !!context.token_payload,
+        errors: ['Authentication required']
+      };
+    },
+    validator: micro.schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
+    handler: async ({ context, params, responder, observer, initialN }) => {
+      const { system } = context;
+
+      if (system.closed) {
+        responder.onError(
+          new micro.errors.JourneyError({
+            status: 503,
+            code: 'SERVICE_UNAVAILABLE',
+            description: 'Service temporarily unavailable'
+          })
+        );
+        responder.onComplete();
+        return;
+      }
+
+      const controller = new AbortController();
+
+      const syncParams: SyncParameters = normalizeTokenParameters(context.token_payload?.parameters ?? {});
+      const storage = system.storage;
+      // Sanity check before we start the stream
+      const cp = await storage.getActiveCheckpoint();
+      if (!cp.hasSyncRules()) {
+        responder.onError(
+          new micro.errors.JourneyError({
+            status: 500,
+            code: 'NO_SYNC_RULES',
+            description: 'No sync rules available'
+          })
+        );
+        responder.onComplete();
+        return;
+      }
+
+      let requestedN = initialN;
+      const disposer = observer.registerListener({
+        request(n) {
+          requestedN += n;
+        },
+        cancel: () => {
+          controller.abort();
+        }
+      });
+
+      const removeStopHandler = system.addStopHandler(() => {
+        observer.triggerCancel();
+      });
+
+      concurrent_connections.add(1);
+      try {
+        for await (const data of streamResponse({
+          storage,
+          params: {
+            ...params,
+            binary_data: true // always true for web sockets
+          },
+          syncParams,
+          token: context!.token_payload!,
+          tokenStreamOptions: {
+            // RSocket handles keepalive events by default
+            keep_alive: false
+          },
+          signal: controller.signal
+        })) {
+          if (data == null) {
+            // Empty value just to flush iterator memory
+            continue;
+          } else if (typeof data == 'string') {
+            // Should not happen with binary_data: true
+            throw new Error(`Unexpected string data: ${data}`);
+          }
+
+          {
+            // On NodeJS, serialize always returns a Buffer
+            const serialized = serialize(data) as Buffer;
+            responder.onNext({ data: serialized }, false);
+            requestedN--;
+            data_synced_bytes.add(serialized.length);
+          }
+
+          if (requestedN <= 0) {
+            await new Promise<void>((resolve) => {
+              const l = observer.registerListener({
+                request() {
+                  if (requestedN > 0) {
+                    // Management of updating the total requested items is done above
+                    resolve();
+                    l();
+                  }
+                },
+                cancel: () => {
+                  // Don't wait here if the request is cancelled
+                  resolve();
+                  l();
+                }
+              });
+            });
+          }
+        }
+      } catch (ex) {
+        // Convert to our standard form before responding.
+        // This ensures the error can be serialized.
+        const error = new micro.errors.InternalServerError(ex);
+        micro.logger.error('Sync stream error', error);
+        responder.onError(error);
+      } finally {
+        responder.onComplete();
+        removeStopHandler();
+        disposer();
+        concurrent_connections.add(-1);
+      }
+    }
+  });

package/src/routes/sync-rules.ts

@@ -0,0 +1,210 @@
+import * as t from 'ts-codec';
+import { FastifyPluginAsync, FastifyReply } from 'fastify';
+import * as micro from '@journeyapps-platform/micro';
+import * as pgwire from '@powersync/service-jpgwire';
+import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
+
+import * as replication from '@/replication/replication-index.js';
+import { authApi } from './auth.js';
+import { RouteGenerator } from './router.js';
+
+const DeploySyncRulesRequest = t.object({
+  content: t.string
+});
+
+const yamlPlugin: FastifyPluginAsync = async (fastify) => {
+  fastify.addContentTypeParser('application/yaml', async (request, payload, _d) => {
+    const data = await micro.streaming.drain(payload);
+
+    request.params = { content: Buffer.concat(data).toString('utf8') };
+  });
+};
+
+export const deploySyncRules: RouteGenerator = (router) =>
+  router.post('/api/sync-rules/v1/deploy', {
+    authorize: authApi,
+    parse: true,
+    plugins: [yamlPlugin],
+    validator: micro.schema.createTsCodecValidator(DeploySyncRulesRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      if (payload.context.system.config.sync_rules.present) {
+        // If sync rules are configured via the config, disable deploy via the API.
+        throw new micro.errors.JourneyError({
+          status: 422,
+          code: 'API_DISABLED',
+          description: 'Sync rules API disabled',
+          details: 'Use the management API to deploy sync rules'
+        });
+      }
+      const content = payload.params.content;
+
+      try {
+        SqlSyncRules.fromYaml(payload.params.content);
+      } catch (e) {
+        throw new micro.errors.JourneyError({
+          status: 422,
+          code: 'INVALID_SYNC_RULES',
+          description: 'Sync rules parsing failed',
+          details: e.message
+        });
+      }
+
+      const sync_rules = await payload.context.system.storage.updateSyncRules({
+        content: content
+      });
+
+      return {
+        slot_name: sync_rules.slot_name
+      };
+    }
+  });
+
+const ValidateSyncRulesRequest = t.object({
+  content: t.string
+});
+
+export const validateSyncRules: RouteGenerator = (router) =>
+  router.post('/api/sync-rules/v1/validate', {
+    authorize: authApi,
+    parse: true,
+    plugins: [yamlPlugin],
+    validator: micro.schema.createTsCodecValidator(ValidateSyncRulesRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const content = payload.params.content;
+
+      const info = await debugSyncRules(payload.context.system.requirePgPool(), content);
+
+      replyPrettyJson(payload.reply, info);
+    }
+  });
+
+export const currentSyncRules: RouteGenerator = (router) =>
+  router.get('/api/sync-rules/v1/current', {
+    authorize: authApi,
+    handler: async (payload) => {
+      const storage = payload.context.system.storage;
+      const sync_rules = await storage.getActiveSyncRulesContent();
+      if (!sync_rules) {
+        throw new micro.errors.JourneyError({
+          status: 422,
+          code: 'NO_SYNC_RULES',
+          description: 'No active sync rules'
+        });
+      }
+      const info = await debugSyncRules(payload.context.system.requirePgPool(), sync_rules.sync_rules_content);
+      const next = await storage.getNextSyncRulesContent();
+
+      const next_info = next
+        ? await debugSyncRules(payload.context.system.requirePgPool(), next.sync_rules_content)
+        : null;
+
+      const response = {
+        current: {
+          slot_name: sync_rules.slot_name,
+          content: sync_rules.sync_rules_content,
+          ...info
+        },
+        next:
+          next == null
+            ? null
+            : {
+                slot_name: next.slot_name,
+                content: next.sync_rules_content,
+                ...next_info
+              }
+      };
+      replyPrettyJson(payload.reply, { data: response });
+    }
+  });
+
+const ReprocessSyncRulesRequest = t.object({});
+
+export const reprocessSyncRules: RouteGenerator = (router) =>
+  router.post('/api/sync-rules/v1/reprocess', {
+    authorize: authApi,
+    validator: micro.schema.createTsCodecValidator(ReprocessSyncRulesRequest),
+    handler: async (payload) => {
+      const storage = payload.context.system.storage;
+      const sync_rules = await storage.getActiveSyncRules();
+      if (sync_rules == null) {
+        throw new micro.errors.JourneyError({
+          status: 422,
+          code: 'NO_SYNC_RULES',
+          description: 'No active sync rules'
+        });
+      }
+
+      const new_rules = await storage.updateSyncRules({
+        content: sync_rules.sync_rules.content
+      });
+      return {
+        slot_name: new_rules.slot_name
+      };
+    }
+  });
+
+export const syncRulesRoutes = [validateSyncRules, deploySyncRules, reprocessSyncRules, currentSyncRules];
+
+function replyPrettyJson(reply: FastifyReply, payload: any) {
+  reply
+    .status(200)
+    .header('Content-Type', 'application/json')
+    .send(JSON.stringify(payload, null, 2) + '\n');
+}
+
+async function debugSyncRules(db: pgwire.PgClient, sync_rules: string) {
+  try {
+    const rules = SqlSyncRules.fromYaml(sync_rules);
+    const source_table_patterns = rules.getSourceTables();
+    const wc = new replication.WalConnection({
+      db: db,
+      sync_rules: rules
+    });
+    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+
+    return {
+      valid: true,
+      bucket_definitions: rules.bucket_descriptors.map((d) => {
+        let all_parameter_queries = [...d.parameter_queries.values()].flat();
+        let all_data_queries = [...d.data_queries.values()].flat();
+        return {
+          name: d.name,
+          bucket_parameters: d.bucket_parameters,
+          global_parameter_queries: d.global_parameter_queries.map((q) => {
+            return {
+              sql: q.sql
+            };
+          }),
+          parameter_queries: all_parameter_queries.map((q) => {
+            return {
+              sql: q.sql,
+              table: q.sourceTable,
+              input_parameters: q.input_parameters
+            };
+          }),
+
+          data_queries: all_data_queries.map((q) => {
+            return {
+              sql: q.sql,
+              table: q.sourceTable,
+              columns: q.columnOutputNames()
+            };
+          })
+        };
+      }),
+      source_tables: resolved_tables,
+      data_tables: rules.debugGetOutputTables()
+    };
+  } catch (e) {
+    if (e instanceof SyncRulesErrors) {
+      return {
+        valid: false,
+        errors: e.errors.map((e) => e.message)
+      };
+    }
+    return {
+      valid: false,
+      errors: [e.message]
+    };
+  }
+}

package/src/routes/sync-stream.ts

@@ -0,0 +1,92 @@
+import { Readable } from 'stream';
+import * as micro from '@journeyapps-platform/micro';
+import { SyncParameters, normalizeTokenParameters } from '@powersync/service-sync-rules';
+
+import * as sync from '@/sync/sync-index.js';
+import * as util from '@/util/util-index.js';
+import { concurrent_connections } from '../metrics/metrics.js';
+
+import { authUser } from './auth.js';
+import { RouteGenerator } from './router.js';
+
+export enum SyncRoutes {
+  STREAM = '/sync/stream'
+}
+
+export const syncStreamed: RouteGenerator = (router) =>
+  router.post(SyncRoutes.STREAM, {
+    authorize: authUser,
+    validator: micro.schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
+    handler: async (payload) => {
+      const userId = payload.context.user_id!;
+      const system = payload.context.system;
+
+      if (system.closed) {
+        throw new micro.errors.JourneyError({
+          status: 503,
+          code: 'SERVICE_UNAVAILABLE',
+          description: 'Service temporarily unavailable'
+        });
+      }
+
+      const params: util.StreamingSyncRequest = payload.params;
+      const syncParams: SyncParameters = normalizeTokenParameters(payload.context.token_payload!.parameters ?? {});
+
+      const storage = system.storage;
+      // Sanity check before we start the stream
+      const cp = await storage.getActiveCheckpoint();
+      if (!cp.hasSyncRules()) {
+        throw new micro.errors.JourneyError({
+          status: 500,
+          code: 'NO_SYNC_RULES',
+          description: 'No sync rules available'
+        });
+      }
+
+      const res = payload.reply;
+
+      res.status(200).header('Content-Type', 'application/x-ndjson');
+
+      const controller = new AbortController();
+      try {
+        concurrent_connections.add(1);
+        const stream = Readable.from(
+          sync.transformToBytesTracked(
+            sync.ndjson(
+              sync.streamResponse({
+                storage,
+                params,
+                syncParams,
+                token: payload.context.token_payload!,
+                signal: controller.signal
+              })
+            )
+          ),
+          { objectMode: false, highWaterMark: 16 * 1024 }
+        );
+
+        const deregister = system.addStopHandler(() => {
+          // This error is not currently propagated to the client
+          controller.abort();
+          stream.destroy(new Error('Shutting down system'));
+        });
+        stream.on('close', () => {
+          deregister();
+        });
+
+        stream.on('error', (error) => {
+          controller.abort();
+          // Note: This appears as a 200 response in the logs.
+          if (error.message != 'Shutting down system') {
+            micro.logger.error('Streaming sync request failed', error);
+          }
+        });
+        await res.send(stream);
+      } finally {
+        controller.abort();
+        concurrent_connections.add(-1);
+        // Prevent double-send
+        res.hijack();
+      }
+    }
+  });

package/src/runner/teardown.ts

@@ -0,0 +1,91 @@
+// Script to tear down the data when deleting an instance.
+// This deletes:
+// 1. The replication slots on the source postgres instance (if available).
+// 2. The mongo database.
+
+import * as micro from '@journeyapps-platform/micro';
+import * as timers from 'timers/promises';
+
+import * as db from '../db/db-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as utils from '../util/util-index.js';
+import * as replication from '../replication/replication-index.js';
+
+/**
+ * Attempt to terminate a single sync rules instance.
+ *
+ * This may fail with a lock error.
+ */
+async function terminateReplicator(
+  storageFactory: storage.BucketStorageFactory,
+  connection: utils.ResolvedConnection,
+  syncRules: storage.PersistedSyncRulesContent
+) {
+  // The lock may still be active if the current replication instance
+  // hasn't stopped yet.
+  const lock = await syncRules.lock();
+  try {
+    const parsed = syncRules.parsed();
+    const storage = storageFactory.getInstance(parsed);
+    const stream = new replication.WalStreamRunner({
+      factory: storageFactory,
+      storage: storage,
+      source_db: connection,
+      lock
+    });
+    console.log('terminating', stream.slot_name);
+    await stream.terminate();
+    console.log('terminated', stream.slot_name);
+  } finally {
+    await lock.release();
+  }
+}
+
+/**
+ * Terminate all replicating sync rules, deleting the replication slots.
+ *
+ * Retries lock and other errors for up to two minutes.
+ *
+ * This is a best-effot attempt. In some cases it may not be possible to delete the replication
+ * slot, such as when the postgres instance is unreachable.
+ */
+async function terminateReplicators(
+  storageFactory: storage.BucketStorageFactory,
+  connection: utils.ResolvedConnection
+) {
+  const start = Date.now();
+  while (Date.now() - start < 12_000) {
+    let retry = false;
+    const replicationRules = await storageFactory.getReplicatingSyncRules();
+    for (let syncRules of replicationRules) {
+      try {
+        await terminateReplicator(storageFactory, connection, syncRules);
+      } catch (e) {
+        retry = true;
+        console.error(e);
+        micro.logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
+      }
+    }
+    if (!retry) {
+      break;
+    }
+    await timers.setTimeout(5_000);
+  }
+}
+
+export async function teardown(runnerConfig: utils.RunnerConfig) {
+  const config = await utils.loadConfig(runnerConfig);
+  const mongoDB = storage.createPowerSyncMongo(config.storage);
+  await db.mongo.waitForAuth(mongoDB.db);
+
+  const bucketStorage = new storage.MongoBucketStorage(mongoDB, { slot_name_prefix: config.slot_name_prefix });
+  const connection = config.connection;
+
+  if (connection) {
+    await terminateReplicators(bucketStorage, connection);
+  }
+
+  const database = mongoDB.db;
+  await database.dropDatabase();
+  await mongoDB.client.close();
+}