@powersync/service-core 0.0.0-dev-20240620165206

package/src/routes/endpoints/admin.ts

@@ -0,0 +1,237 @@
+import { errors, router, schema } from '@powersync/lib-services-framework';
+import { SqlSyncRules, SqliteValue, StaticSchema, isJsonValue, toSyncRulesValue } from '@powersync/service-sync-rules';
+import { internal_routes } from '@powersync/service-types';
+
+import * as api from '../../api/api-index.js';
+import * as util from '../../util/util-index.js';
+
+import { routeDefinition } from '../router.js';
+import { PersistedSyncRulesContent } from '../../storage/BucketStorage.js';
+import { authApi } from '../auth.js';
+
+const demoCredentials = routeDefinition({
+  path: '/api/admin/v1/demo-credentials',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.DemoCredentialsRequest, {
+    allowAdditional: true
+  }),
+  handler: async (payload) => {
+    const connection = payload.context.system.config.connection;
+    if (connection == null || !connection.demo_database) {
+      return internal_routes.DemoCredentialsResponse.encode({});
+    }
+
+    const uri = util.buildDemoPgUri(connection);
+    return internal_routes.DemoCredentialsResponse.encode({
+      credentials: {
+        postgres_uri: uri
+      }
+    });
+  }
+});
+
+export const executeSql = routeDefinition({
+  path: '/api/admin/v1/execute-sql',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.ExecuteSqlRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const connection = payload.context.system.config.connection;
+    if (connection == null || !connection.debug_api) {
+      return internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: 'SQL querying is not enabled'
+      });
+    }
+
+    const pool = payload.context.system.requirePgPool();
+
+    const { query, args } = payload.params.sql;
+
+    try {
+      const result = await pool.query({
+        statement: query,
+        params: args.map(util.autoParameter)
+      });
+
+      return internal_routes.ExecuteSqlResponse.encode({
+        success: true,
+        results: {
+          columns: result.columns.map((c) => c.name),
+          rows: result.rows.map((row) => {
+            return row.map((value) => mapColumnValue(toSyncRulesValue(value)));
+          })
+        }
+      });
+    } catch (e) {
+      return internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: e.message
+      });
+    }
+  }
+});
+
+export const diagnostics = routeDefinition({
+  path: '/api/admin/v1/diagnostics',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.DiagnosticsRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const include_content = payload.params.sync_rules_content ?? false;
+    const system = payload.context.system;
+
+    const status = await api.getConnectionStatus(system);
+    if (status == null) {
+      return internal_routes.DiagnosticsResponse.encode({
+        connections: []
+      });
+    }
+
+    const { storage } = system;
+    const active = await storage.getActiveSyncRulesContent();
+    const next = await storage.getNextSyncRulesContent();
+
+    const active_status = await api.getSyncRulesStatus(active, system, {
+      include_content,
+      check_connection: status.connected,
+      live_status: true
+    });
+
+    const next_status = await api.getSyncRulesStatus(next, system, {
+      include_content,
+      check_connection: status.connected,
+      live_status: true
+    });
+
+    return internal_routes.DiagnosticsResponse.encode({
+      connections: [status],
+      active_sync_rules: active_status,
+      deploying_sync_rules: next_status
+    });
+  }
+});
+
+export const getSchema = routeDefinition({
+  path: '/api/admin/v1/schema',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.GetSchemaRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    return internal_routes.GetSchemaResponse.encode(await api.getConnectionsSchema(system));
+  }
+});
+
+export const reprocess = routeDefinition({
+  path: '/api/admin/v1/reprocess',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.ReprocessRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    const storage = system.storage;
+    const next = await storage.getNextSyncRules();
+    if (next != null) {
+      throw new Error(`Busy processing sync rules - cannot reprocess`);
+    }
+
+    const active = await storage.getActiveSyncRules();
+    if (active == null) {
+      throw new errors.JourneyError({
+        status: 422,
+        code: 'NO_SYNC_RULES',
+        description: 'No active sync rules'
+      });
+    }
+
+    const new_rules = await storage.updateSyncRules({
+      content: active.sync_rules.content
+    });
+
+    return internal_routes.ReprocessResponse.encode({
+      connections: [
+        {
+          tag: system.config.connection!.tag,
+          id: system.config.connection!.id,
+          slot_name: new_rules.slot_name
+        }
+      ]
+    });
+  }
+});
+
+export const validate = routeDefinition({
+  path: '/api/admin/v1/validate',
+  method: router.HTTPMethod.POST,
+  authorize: authApi,
+  validator: schema.createTsCodecValidator(internal_routes.ValidateRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+
+    const content = payload.params.sync_rules;
+
+    const schemaData = await api.getConnectionsSchema(system);
+    const schema = new StaticSchema(schemaData.connections);
+
+    const sync_rules: PersistedSyncRulesContent = {
+      // Dummy values
+      id: 0,
+      slot_name: '',
+
+      parsed() {
+        return {
+          ...this,
+          sync_rules: SqlSyncRules.fromYaml(content, { throwOnError: false, schema })
+        };
+      },
+      sync_rules_content: content,
+      async lock() {
+        throw new Error('Lock not implemented');
+      }
+    };
+
+    const connectionStatus = await api.getConnectionStatus(system);
+    if (connectionStatus == null) {
+      return internal_routes.ValidateResponse.encode({
+        errors: [{ level: 'fatal', message: 'No connection configured' }],
+        connections: []
+      });
+    }
+
+    const status = (await api.getSyncRulesStatus(sync_rules, system, {
+      include_content: false,
+      check_connection: connectionStatus?.connected,
+      live_status: false
+    }))!;
+
+    if (connectionStatus == null) {
+      status.errors.push({ level: 'fatal', message: 'No connection configured' });
+    }
+
+    return internal_routes.ValidateResponse.encode(status);
+  }
+});
+
+function mapColumnValue(value: SqliteValue) {
+  if (typeof value == 'bigint') {
+    return Number(value);
+  } else if (isJsonValue(value)) {
+    return value;
+  } else {
+    return null;
+  }
+}
+
+export const ADMIN_ROUTES = [demoCredentials, executeSql, diagnostics, getSchema, reprocess, validate];

package/src/routes/endpoints/checkpointing.ts

@@ -0,0 +1,41 @@
+import * as t from 'ts-codec';
+import { router, schema } from '@powersync/lib-services-framework';
+
+import * as util from '../../util/util-index.js';
+import { authUser } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+const WriteCheckpointRequest = t.object({});
+
+export const writeCheckpoint = routeDefinition({
+  path: '/write-checkpoint.json',
+  method: router.HTTPMethod.GET,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const system = payload.context.system;
+    const storage = system.storage;
+
+    const checkpoint = await util.getClientCheckpoint(system.requirePgPool(), storage);
+    return {
+      checkpoint
+    };
+  }
+});
+
+export const writeCheckpoint2 = routeDefinition({
+  path: '/write-checkpoint2.json',
+  method: router.HTTPMethod.GET,
+  authorize: authUser,
+  validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
+  handler: async (payload) => {
+    const { user_id, system } = payload.context;
+    const storage = system.storage;
+    const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, user_id!);
+    return {
+      write_checkpoint: String(write_checkpoint)
+    };
+  }
+});
+
+export const CHECKPOINT_ROUTES = [writeCheckpoint, writeCheckpoint2];

package/src/routes/endpoints/dev.ts

@@ -0,0 +1,199 @@
+import * as t from 'ts-codec';
+import * as pgwire from '@powersync/service-jpgwire';
+import { errors, router, schema } from '@powersync/lib-services-framework';
+
+import * as util from '../../util/util-index.js';
+import { authDevUser, authUser, endpoint, issueDevToken, issueLegacyDevToken, issuePowerSyncToken } from '../auth.js';
+import { routeDefinition } from '../router.js';
+
+const AuthParams = t.object({
+  user: t.string,
+  password: t.string
+});
+
+// For legacy web client only. Remove soon.
+export const auth = routeDefinition({
+  path: '/auth.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
+  handler: async (payload) => {
+    const { user, password } = payload.params;
+    const config = payload.context.system.config;
+
+    if (config.dev.demo_auth == false || config.dev.demo_password == null) {
+      throw new errors.AuthorizationError(['Demo auth disabled']);
+    }
+
+    if (password == config.dev.demo_password) {
+      const token = await issueLegacyDevToken(payload.request, user, payload.context.system.config);
+      return { token, user_id: user, endpoint: endpoint(payload.request) };
+    } else {
+      throw new errors.AuthorizationError(['Authentication failed']);
+    }
+  }
+});
+
+export const auth2 = routeDefinition({
+  path: '/dev/auth.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(AuthParams, { allowAdditional: true }),
+  handler: async (payload) => {
+    const { user, password } = payload.params;
+    const config = payload.context.system.config;
+
+    if (config.dev.demo_auth == false || config.dev.demo_password == null) {
+      throw new errors.AuthorizationError(['Demo auth disabled']);
+    }
+
+    if (password == config.dev.demo_password) {
+      const token = await issueDevToken(payload.request, user, payload.context.system.config);
+      return { token, user_id: user };
+    } else {
+      throw new errors.AuthorizationError(['Authentication failed']);
+    }
+  }
+});
+
+const TokenParams = t.object({});
+
+export const token = routeDefinition({
+  path: '/dev/token.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(TokenParams, { allowAdditional: true }),
+  authorize: authDevUser,
+  handler: async (payload) => {
+    const { user_id } = payload.context;
+    const outToken = await issuePowerSyncToken(payload.request, user_id!, payload.context.system.config);
+    return { token: outToken, user_id: user_id, endpoint: endpoint(payload.request) };
+  }
+});
+
+const OpType = {
+  PUT: 'PUT',
+  PATCH: 'PATCH',
+  DELETE: 'DELETE'
+};
+
+const CrudEntry = t.object({
+  op: t.Enum(OpType),
+  type: t.string,
+  id: t.string,
+  op_id: t.number.optional(),
+  data: t.any.optional()
+});
+
+const CrudRequest = t.object({
+  data: t.array(CrudEntry),
+  write_checkpoint: t.boolean.optional()
+});
+
+export const crud = routeDefinition({
+  path: '/crud.json',
+  method: router.HTTPMethod.POST,
+  validator: schema.createTsCodecValidator(CrudRequest, { allowAdditional: true }),
+  authorize: authUser,
+
+  handler: async (payload) => {
+    const { user_id, system } = payload.context;
+
+    const pool = system.requirePgPool();
+
+    if (!system.config.dev.crud_api) {
+      throw new Error('CRUD api disabled');
+    }
+
+    const params = payload.params;
+
+    let statements: pgwire.Statement[] = [];
+
+    // Implementation note:
+    // Postgres does automatic "assigment cast" for query literals,
+    // e.g. a string literal to uuid. However, the same doesn't apply
+    // to query parameters.
+    // To handle those automatically, we use `json_populate_record`
+    // to automatically cast to the correct types.
+
+    for (let op of params.data) {
+      const table = util.escapeIdentifier(op.type);
+      if (op.op == 'PUT') {
+        const data = op.data as Record<string, any>;
+        const with_id = { ...data, id: op.id };
+
+        const columnsEscaped = Object.keys(with_id).map(util.escapeIdentifier);
+        const columnsJoined = columnsEscaped.join(', ');
+
+        let updateClauses: string[] = [];
+
+        for (let key of Object.keys(data)) {
+          updateClauses.push(`${util.escapeIdentifier(key)} = EXCLUDED.${util.escapeIdentifier(key)}`);
+        }
+
+        const updateClause = updateClauses.length > 0 ? `DO UPDATE SET ${updateClauses.join(', ')}` : `DO NOTHING`;
+
+        const statement = `
+          WITH data_row AS (
+              SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          INSERT INTO ${table} (${columnsJoined})
+          SELECT ${columnsJoined} FROM data_row
+          ON CONFLICT(id) ${updateClause}`;
+
+        statements.push({
+          statement: statement,
+          params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
+        });
+      } else if (op.op == 'PATCH') {
+        const data = op.data as Record<string, any>;
+        const with_id = { ...data, id: op.id };
+
+        let updateClauses: string[] = [];
+
+        for (let key of Object.keys(data)) {
+          updateClauses.push(`${util.escapeIdentifier(key)} = data_row.${util.escapeIdentifier(key)}`);
+        }
+
+        const statement = `
+          WITH data_row AS (
+              SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          UPDATE ${table}
+          SET ${updateClauses.join(', ')}
+          FROM data_row
+          WHERE ${table}.id = data_row.id`;
+
+        statements.push({
+          statement: statement,
+          params: [{ type: 'varchar', value: JSON.stringify(with_id) }]
+        });
+      } else if (op.op == 'DELETE') {
+        statements.push({
+          statement: `
+          WITH data_row AS (
+            SELECT (json_populate_record(null::${table}, $1::json)).*
+          )
+          DELETE FROM ${table}
+          USING data_row
+          WHERE ${table}.id = data_row.id`,
+          params: [{ type: 'varchar', value: JSON.stringify({ id: op.id }) }]
+        });
+      }
+    }
+    await pool.query(...statements);
+
+    const storage = system.storage;
+    if (payload.params.write_checkpoint === true) {
+      const write_checkpoint = await util.createWriteCheckpoint(pool, storage, payload.context.user_id!);
+      return { write_checkpoint: String(write_checkpoint) };
+    } else if (payload.params.write_checkpoint === false) {
+      return {};
+    } else {
+      // Legacy
+      const checkpoint = await util.getClientCheckpoint(pool, storage);
+      return {
+        checkpoint
+      };
+    }
+  }
+});
+
+export const DEV_ROUTES = [auth, auth2, token, crud];

package/src/routes/endpoints/route-endpoints-index.ts

@@ -0,0 +1,6 @@
+export * from './admin.js';
+export * from './checkpointing.js';
+export * from './dev.js';
+export * from './socket-route.js';
+export * from './sync-rules.js';
+export * from './sync-stream.js';

package/src/routes/endpoints/socket-route.ts

@@ -0,0 +1,135 @@
+import { serialize } from 'bson';
+import { SyncParameters, normalizeTokenParameters } from '@powersync/service-sync-rules';
+import { errors, logger, schema } from '@powersync/lib-services-framework';
+
+import * as util from '../../util/util-index.js';
+import { streamResponse } from '../../sync/sync.js';
+import { SyncRoutes } from './sync-stream.js';
+import { SocketRouteGenerator } from '../router-socket.js';
+import { Metrics } from '../../metrics/Metrics.js';
+
+export const syncStreamReactive: SocketRouteGenerator = (router) =>
+  router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
+    authorize: ({ context }) => {
+      return {
+        authorized: !!context.token_payload,
+        errors: ['Authentication required'].concat(context.token_errors ?? [])
+      };
+    },
+    validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
+    handler: async ({ context, params, responder, observer, initialN }) => {
+      const { system } = context;
+
+      if (system.closed) {
+        responder.onError(
+          new errors.JourneyError({
+            status: 503,
+            code: 'SERVICE_UNAVAILABLE',
+            description: 'Service temporarily unavailable'
+          })
+        );
+        responder.onComplete();
+        return;
+      }
+
+      const controller = new AbortController();
+
+      const syncParams: SyncParameters = normalizeTokenParameters(
+        context.token_payload?.parameters ?? {},
+        params.parameters ?? {}
+      );
+
+      const storage = system.storage;
+      // Sanity check before we start the stream
+      const cp = await storage.getActiveCheckpoint();
+      if (!cp.hasSyncRules()) {
+        responder.onError(
+          new errors.JourneyError({
+            status: 500,
+            code: 'NO_SYNC_RULES',
+            description: 'No sync rules available'
+          })
+        );
+        responder.onComplete();
+        return;
+      }
+
+      let requestedN = initialN;
+      const disposer = observer.registerListener({
+        request(n) {
+          requestedN += n;
+        },
+        cancel: () => {
+          controller.abort();
+        }
+      });
+
+      const removeStopHandler = system.addStopHandler(() => {
+        observer.triggerCancel();
+      });
+
+      Metrics.getInstance().concurrent_connections.add(1);
+      try {
+        for await (const data of streamResponse({
+          storage,
+          params: {
+            ...params,
+            binary_data: true // always true for web sockets
+          },
+          syncParams,
+          token: context!.token_payload!,
+          tokenStreamOptions: {
+            // RSocket handles keepalive events by default
+            keep_alive: false
+          },
+          signal: controller.signal
+        })) {
+          if (data == null) {
+            // Empty value just to flush iterator memory
+            continue;
+          } else if (typeof data == 'string') {
+            // Should not happen with binary_data: true
+            throw new Error(`Unexpected string data: ${data}`);
+          }
+
+          {
+            // On NodeJS, serialize always returns a Buffer
+            const serialized = serialize(data) as Buffer;
+            responder.onNext({ data: serialized }, false);
+            requestedN--;
+            Metrics.getInstance().data_synced_bytes.add(serialized.length);
+          }
+
+          if (requestedN <= 0) {
+            await new Promise<void>((resolve) => {
+              const l = observer.registerListener({
+                request() {
+                  if (requestedN > 0) {
+                    // Management of updating the total requested items is done above
+                    resolve();
+                    l();
+                  }
+                },
+                cancel: () => {
+                  // Don't wait here if the request is cancelled
+                  resolve();
+                  l();
+                }
+              });
+            });
+          }
+        }
+      } catch (ex) {
+        // Convert to our standard form before responding.
+        // This ensures the error can be serialized.
+        const error = new errors.InternalServerError(ex);
+        logger.error('Sync stream error', error);
+        responder.onError(error);
+      } finally {
+        responder.onComplete();
+        removeStopHandler();
+        disposer();
+        Metrics.getInstance().concurrent_connections.add(-1);
+      }
+    }
+  });