@powersync/service-core 0.0.0-dev-20240620165206

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@powersync/service-core",
+  "repository": "https://github.com/powersync-ja/powersync-service",
+  "types": "dist/index.d.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "version": "0.0.0-dev-20240620165206",
+  "main": "dist/index.js",
+  "license": "FSL-1.1-Apache-2.0",
+  "type": "module",
+  "dependencies": {
+    "@js-sdsl/ordered-set": "^4.4.2",
+    "@opentelemetry/api": "~1.8.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.51.1",
+    "@opentelemetry/exporter-prometheus": "^0.51.1",
+    "@opentelemetry/resources": "^1.24.1",
+    "@opentelemetry/sdk-metrics": "1.24.1",
+    "async": "^3.2.4",
+    "async-mutex": "^0.5.0",
+    "bson": "^6.6.0",
+    "commander": "^12.0.0",
+    "cors": "^2.8.5",
+    "ipaddr.js": "^2.1.0",
+    "ix": "^5.0.0",
+    "jose": "^4.15.1",
+    "lodash": "^4.17.21",
+    "lru-cache": "^10.2.2",
+    "mongodb": "^6.7.0",
+    "node-fetch": "^3.3.2",
+    "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
+    "ts-codec": "^1.2.2",
+    "uuid": "^9.0.1",
+    "winston": "^3.13.0",
+    "yaml": "^2.3.2",
+    "@powersync/lib-services-framework": "0.0.0-dev-20240620165206",
+    "@powersync/service-jpgwire": "0.17.13",
+    "@powersync/service-jsonbig": "0.17.10",
+    "@powersync/service-rsocket-router": "0.0.0-dev-20240620165206",
+    "@powersync/service-sync-rules": "0.17.10",
+    "@powersync/service-types": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/async": "^3.2.24",
+    "@types/lodash": "^4.17.5",
+    "@types/uuid": "^9.0.4",
+    "fastify": "4.23.2",
+    "fastify-plugin": "^4.5.1",
+    "typescript": "^5.2.2",
+    "vite-tsconfig-paths": "^4.3.2",
+    "vitest": "^0.34.6"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "build:tests": "tsc -b test/tsconfig.json",
+    "test": "vitest --no-threads",
+    "clean": "rm -rf ./lib && tsc -b --clean"
+  }
+}

package/src/api/api-index.ts

@@ -0,0 +1,2 @@
+export * from './diagnostics.js';
+export * from './schema.js';

package/src/api/diagnostics.ts

@@ -0,0 +1,221 @@
+import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules } from '@powersync/service-sync-rules';
+import { pgwireRows } from '@powersync/service-jpgwire';
+import { ConnectionStatus, SyncRulesStatus, TableInfo, baseUri } from '@powersync/service-types';
+
+import * as replication from '../replication/replication-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
+
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { logger } from '@powersync/lib-services-framework';
+
+export async function getConnectionStatus(system: CorePowerSyncSystem): Promise<ConnectionStatus | null> {
+  if (system.pgwire_pool == null) {
+    return null;
+  }
+
+  const pool = system.requirePgPool();
+
+  const base = {
+    id: system.config.connection!.id,
+    postgres_uri: baseUri(system.config.connection!)
+  };
+  try {
+    await util.retriedQuery(pool, `SELECT 'PowerSync connection test'`);
+  } catch (e) {
+    return {
+      ...base,
+      connected: false,
+      errors: [{ level: 'fatal', message: e.message }]
+    };
+  }
+
+  try {
+    await replication.checkSourceConfiguration(pool);
+  } catch (e) {
+    return {
+      ...base,
+      connected: true,
+      errors: [{ level: 'fatal', message: e.message }]
+    };
+  }
+
+  return {
+    ...base,
+    connected: true,
+    errors: []
+  };
+}
+
+export interface DiagnosticsOptions {
+  /**
+   * Include sync rules content in response.
+   */
+  include_content?: boolean;
+
+  /**
+   * Check against storage database.
+   *
+   * If false, uses placeholder values for e.g. initial_replication_done.
+   */
+  live_status: boolean;
+
+  /**
+   * Check against the source postgres connection.
+   */
+  check_connection: boolean;
+}
+
+export async function getSyncRulesStatus(
+  sync_rules: storage.PersistedSyncRulesContent | null,
+  system: CorePowerSyncSystem,
+  options: DiagnosticsOptions
+): Promise<SyncRulesStatus | undefined> {
+  if (sync_rules == null) {
+    return undefined;
+  }
+
+  const include_content = options.include_content ?? false;
+  const live_status = options.live_status ?? false;
+  const check_connection = options.check_connection ?? false;
+
+  let rules: SqlSyncRules;
+  let persisted: storage.PersistedSyncRules;
+  try {
+    persisted = sync_rules.parsed();
+    rules = persisted.sync_rules;
+  } catch (e) {
+    return {
+      content: include_content ? sync_rules.sync_rules_content : undefined,
+      connections: [],
+      errors: [{ level: 'fatal', message: e.message }]
+    };
+  }
+
+  const systemStorage = live_status ? await system.storage.getInstance(persisted) : undefined;
+  const status = await systemStorage?.getStatus();
+  let replication_lag_bytes: number | undefined = undefined;
+
+  let tables_flat: TableInfo[] = [];
+
+  if (check_connection) {
+    const pool = system.requirePgPool();
+
+    const source_table_patterns = rules.getSourceTables();
+    const wc = new replication.WalConnection({
+      db: pool,
+      sync_rules: rules
+    });
+    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+    tables_flat = resolved_tables.flatMap((info) => {
+      if (info.table) {
+        return [info.table];
+      } else if (info.tables) {
+        return info.tables;
+      } else {
+        return [];
+      }
+    });
+
+    if (systemStorage) {
+      try {
+        const results = await util.retriedQuery(pool, {
+          statement: `SELECT
+      slot_name,
+      confirmed_flush_lsn, 
+      pg_current_wal_lsn(), 
+      (pg_current_wal_lsn() - confirmed_flush_lsn) AS lsn_distance
+    FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
+          params: [{ type: 'varchar', value: systemStorage!.slot_name }]
+        });
+        const [row] = pgwireRows(results);
+        if (row) {
+          replication_lag_bytes = Number(row.lsn_distance);
+        }
+      } catch (e) {
+        // Ignore
+        logger.warn(`Unable to get replication lag`, e);
+      }
+    }
+  } else {
+    const source_table_patterns = rules.getSourceTables();
+    const tag = system.config.connection!.tag ?? DEFAULT_TAG;
+
+    tables_flat = source_table_patterns.map((pattern): TableInfo => {
+      if (pattern.isWildcard) {
+        return {
+          schema: pattern.schema,
+          name: pattern.tablePrefix,
+          pattern: pattern.isWildcard ? pattern.tablePattern : undefined,
+
+          data_queries: false,
+          parameter_queries: false,
+          replication_id: [],
+          errors: [{ level: 'fatal', message: 'connection failed' }]
+        };
+      } else {
+        const source: SourceTableInterface = {
+          connectionTag: tag,
+          schema: pattern.schema,
+          table: pattern.tablePattern
+        };
+        const syncData = rules.tableSyncsData(source);
+        const syncParameters = rules.tableSyncsParameters(source);
+        return {
+          schema: pattern.schema,
+          name: pattern.name,
+          data_queries: syncData,
+          parameter_queries: syncParameters,
+          replication_id: [],
+          errors: [{ level: 'fatal', message: 'connection failed' }]
+        };
+      }
+    });
+  }
+
+  const errors = tables_flat.flatMap((info) => info.errors);
+  if (sync_rules.last_fatal_error) {
+    errors.push({ level: 'fatal', message: sync_rules.last_fatal_error });
+  }
+  errors.push(
+    ...rules.errors.map((e) => {
+      return {
+        level: e.type,
+        message: e.message
+      };
+    })
+  );
+
+  return {
+    content: include_content ? sync_rules.sync_rules_content : undefined,
+    connections: [
+      {
+        id: system.config.connection!.id,
+        tag: system.config.connection!.tag ?? DEFAULT_TAG,
+        slot_name: sync_rules.slot_name,
+        initial_replication_done: status?.snapshot_done ?? false,
+        // TODO: Rename?
+        last_lsn: status?.checkpoint_lsn ?? undefined,
+        last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
+        last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
+        replication_lag_bytes: replication_lag_bytes,
+        tables: tables_flat
+      }
+    ],
+    errors: deduplicate(errors)
+  };
+}
+
+function deduplicate(errors: { level: 'warning' | 'fatal'; message: string }[]) {
+  let seen = new Set<string>();
+  let result: { level: 'warning' | 'fatal'; message: string }[] = [];
+  for (let error of errors) {
+    const key = JSON.stringify(error);
+    if (seen.has(key)) {
+      continue;
+    }
+    seen.add(key);
+    result.push(error);
+  }
+  return result;
+}

package/src/api/schema.ts

@@ -0,0 +1,99 @@
+import type * as pgwire from '@powersync/service-jpgwire';
+import { pgwireRows } from '@powersync/service-jpgwire';
+import { DatabaseSchema, internal_routes } from '@powersync/service-types';
+
+import * as util from '../util/util-index.js';
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+
+export async function getConnectionsSchema(system: CorePowerSyncSystem): Promise<internal_routes.GetSchemaResponse> {
+  if (system.config.connection == null) {
+    return { connections: [] };
+  }
+  const schemas = await getConnectionSchema(system.requirePgPool());
+  return {
+    connections: [
+      {
+        schemas,
+        tag: system.config.connection!.tag,
+        id: system.config.connection!.id
+      }
+    ]
+  };
+}
+
+export async function getConnectionSchema(db: pgwire.PgClient): Promise<DatabaseSchema[]> {
+  // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
+  const results = await util.retriedQuery(
+    db,
+    `SELECT
+  tbl.schemaname,
+  tbl.tablename,
+  tbl.quoted_name,
+  json_agg(a ORDER BY attnum) as columns
+FROM
+  (
+    SELECT
+      n.nspname as schemaname,
+      c.relname as tablename,
+      (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) as quoted_name
+    FROM
+      pg_catalog.pg_class c
+      JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+    WHERE
+      c.relkind = 'r'
+      AND n.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
+      AND n.nspname not like 'pg_temp_%'
+      AND n.nspname not like 'pg_toast_temp_%'
+      AND c.relnatts > 0
+      AND has_schema_privilege(n.oid, 'USAGE') = true
+      AND has_table_privilege(quote_ident(n.nspname) || '.' || quote_ident(c.relname), 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER') = true
+  ) as tbl
+  LEFT JOIN (
+    SELECT
+      attrelid,
+      attname,
+      format_type(atttypid, atttypmod) as data_type,
+      (SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type,
+      attnum,
+      attisdropped
+    FROM
+      pg_attribute
+  ) as a ON (
+    a.attrelid = tbl.quoted_name::regclass
+    AND a.attnum > 0
+    AND NOT a.attisdropped
+    AND has_column_privilege(tbl.quoted_name, a.attname, 'SELECT, INSERT, UPDATE, REFERENCES')
+  )
+GROUP BY schemaname, tablename, quoted_name`
+  );
+  const rows = pgwireRows(results);
+
+  let schemas: Record<string, any> = {};
+
+  for (let row of rows) {
+    const schema = (schemas[row.schemaname] ??= {
+      name: row.schemaname,
+      tables: []
+    });
+    const table = {
+      name: row.tablename,
+      columns: [] as any[]
+    };
+    schema.tables.push(table);
+
+    const columnInfo = JSON.parse(row.columns);
+    for (let column of columnInfo) {
+      let pg_type = column.pg_type as string;
+      if (pg_type.startsWith('_')) {
+        pg_type = `${pg_type.substring(1)}[]`;
+      }
+      table.columns.push({
+        name: column.attname,
+        type: column.data_type,
+        pg_type: pg_type
+      });
+    }
+  }
+
+  return Object.values(schemas);
+}

package/src/auth/CachedKeyCollector.ts

@@ -0,0 +1,132 @@
+import * as jose from 'jose';
+import timers from 'timers/promises';
+import { KeySpec } from './KeySpec.js';
+import { LeakyBucket } from './LeakyBucket.js';
+import { KeyCollector, KeyResult } from './KeyCollector.js';
+
+/**
+ * Manages caching and refreshing for a key collector.
+ *
+ * Cache refreshing is activity-based, instead of automatically refreshing.
+ *
+ * Generally:
+ *  * If the last refresh was > 5 minutes ago, trigger a background refresh, but use the cached keys.
+ *  * If the last refresh was > 60 minutes ago, discard the cache, and force a refresh.
+ *  * If the last refresh resulted in an error, refresh based on a retry delay, but use cached keys.
+ */
+
+export class CachedKeyCollector implements KeyCollector {
+  private currentKeys: KeySpec[] = [];
+  /**
+   * The time that currentKeys was set.
+   */
+  private keyTimestamp: number = 0;
+
+  /**
+   * Refresh every 5 minutes - the default refresh rate.
+   */
+  private backgroundRefreshInterval = 300000;
+
+  /**
+   * Refresh a _max_ of once every minute at steady state.
+   *
+   * This controls the refresh rate under error conditions.
+   */
+  private rateLimiter = new LeakyBucket({ maxCapacity: 10, periodMs: 60000 });
+
+  /**
+   * Expire keys after an hour, if we failed to refresh in that time.
+   */
+  private keyExpiry = 3600000;
+
+  private currentErrors: jose.errors.JOSEError[] = [];
+  /**
+   * Indicates a "fatal" error that should be retried.
+   */
+  private error = false;
+
+  private refreshPromise: Promise<void> | undefined = undefined;
+
+  constructor(private source: KeyCollector) {}
+
+  async getKeys(): Promise<KeyResult> {
+    const now = Date.now();
+    if (now - this.keyTimestamp > this.keyExpiry) {
+      // Keys have expired - clear
+      this.currentKeys = [];
+    }
+
+    if (this.wantsRefresh()) {
+      // Trigger background refresh.
+      // This also sets refreshPromise
+      this.refresh();
+    }
+
+    if (now - this.keyTimestamp > this.keyExpiry) {
+      // Keys have expired - wait for fetching new keys
+      // It is possible that the refresh was actually triggered,
+      // e.g. in the case of waiting for error retries.
+      // In the case of very slow requests, we don't wait for it to complete, but the
+      // request can still complete in the background.
+      const timeout = timers.setTimeout(3000);
+      await Promise.race([this.refreshPromise, timeout]);
+    }
+
+    return { keys: this.currentKeys, errors: this.currentErrors };
+  }
+
+  private refresh() {
+    if (this.refreshPromise == null) {
+      if (!this.rateLimiter.allowed()) {
+        return;
+      }
+      this.refreshPromise = this.refreshInner().finally(() => {
+        this.refreshPromise = undefined;
+      });
+    }
+    return this.refreshPromise;
+  }
+
+  async noKeyFound(): Promise<void> {
+    // Refresh keys if allowed by the rate limiter
+    await this.refresh();
+  }
+
+  private async refreshInner() {
+    try {
+      const { keys, errors } = await this.source.getKeys();
+      // Partial or full result
+      this.currentKeys = keys;
+      this.currentErrors = errors;
+      this.keyTimestamp = Date.now();
+      this.error = false;
+    } catch (e) {
+      this.error = true;
+      // No result - keep previous keys
+      if (e instanceof jose.errors.JOSEError) {
+        this.currentErrors = [e];
+      } else {
+        this.currentErrors = [new jose.errors.JOSEError(e.message ?? 'Failed to fetch keys')];
+      }
+    }
+  }
+
+  private wantsRefresh() {
+    if (this.error) {
+      return true;
+    }
+
+    if (Date.now() - this.rateLimiter.lastGrantedRequest >= this.backgroundRefreshInterval) {
+      return true;
+    }
+
+    return false;
+  }
+
+  async addTimeForTests(time: number) {
+    this.keyTimestamp -= time;
+    this.rateLimiter.reset();
+    this.rateLimiter.lastGrantedRequest -= time;
+    await this.refreshPromise;
+  }
+}

package/src/auth/CompoundKeyCollector.ts

@@ -0,0 +1,33 @@
+import * as jose from 'jose';
+import { KeySpec } from './KeySpec.js';
+import { KeyCollector, KeyResult } from './KeyCollector.js';
+
+export class CompoundKeyCollector implements KeyCollector {
+  private collectors: KeyCollector[];
+
+  constructor(collectors?: KeyCollector[]) {
+    this.collectors = collectors ?? [];
+  }
+
+  add(collector: KeyCollector) {
+    this.collectors.push(collector);
+  }
+
+  async getKeys(): Promise<KeyResult> {
+    let keys: KeySpec[] = [];
+    let errors: jose.errors.JOSEError[] = [];
+    const promises = this.collectors.map((collector) =>
+      collector.getKeys().then((result) => {
+        keys.push(...result.keys);
+        errors.push(...result.errors);
+      })
+    );
+    await Promise.all(promises);
+    return { keys, errors };
+  }
+
+  async noKeyFound(): Promise<void> {
+    const promises = this.collectors.map((collector) => collector.noKeyFound?.());
+    await Promise.all(promises);
+  }
+}

package/src/auth/JwtPayload.ts

@@ -0,0 +1,11 @@
+export interface JwtPayload {
+  /**
+   * user_id
+   */
+  sub: string;
+  parameters: Record<string, any>;
+
+  iss?: string | undefined;
+  exp: number;
+  iat: number;
+}

package/src/auth/KeyCollector.ts

@@ -0,0 +1,27 @@
+import * as jose from 'jose';
+import { KeySpec } from './KeySpec.js';
+
+export interface KeyCollector {
+  /**
+   * Fetch keys for this collector.
+   *
+   * If a partial result is available, return keys and errors array.
+   * These errors are not retried, and previous keys not cached.
+   *
+   * If the request fails completely, throw an error. These errors are retried.
+   * In that case, previous keys may be cached and used.
+   */
+  getKeys(): Promise<KeyResult>;
+
+  /**
+   * Indicates that no matching key was found.
+   *
+   * The collector may use this as a hint to reload keys, although this is not a requirement.
+   */
+  noKeyFound?: () => Promise<void>;
+}
+
+export interface KeyResult {
+  errors: jose.errors.JOSEError[];
+  keys: KeySpec[];
+}

package/src/auth/KeySpec.ts

@@ -0,0 +1,67 @@
+import * as jose from 'jose';
+
+export const HS_ALGORITHMS = ['HS256', 'HS384', 'HS512'];
+export const RSA_ALGORITHMS = ['RS256', 'RS384', 'RS512'];
+export const SUPPORTED_ALGORITHMS = [...HS_ALGORITHMS, ...RSA_ALGORITHMS];
+
+export interface KeyOptions {
+  /**
+   * If configured, JWTs verified by this key must have one of these audiences
+   * in the `aud` claim, instead of the default.
+   */
+  requiresAudience?: string[];
+
+  /**
+   * If configured, JWTs verified by this key can have a maximum lifetime up to
+   * this value, instead of the default.
+   */
+  maxLifetimeSeconds?: number;
+}
+
+export class KeySpec {
+  key: jose.KeyLike;
+  source: jose.JWK;
+  options: KeyOptions;
+
+  static async importKey(key: jose.JWK, options?: KeyOptions): Promise<KeySpec> {
+    const parsed = (await jose.importJWK(key)) as jose.KeyLike;
+    return new KeySpec(key, parsed, options);
+  }
+
+  constructor(source: jose.JWK, key: jose.KeyLike, options?: KeyOptions) {
+    this.source = source;
+    this.key = key;
+    this.options = options ?? {};
+  }
+
+  get kid(): string | undefined {
+    return this.source.kid;
+  }
+
+  matchesAlgorithm(jwtAlg: string) {
+    if (this.source.alg) {
+      return jwtAlg == this.source.alg;
+    } else if (this.source.kty == 'RSA') {
+      return RSA_ALGORITHMS.includes(jwtAlg);
+    } else if (this.source.kty == 'oct') {
+      return HS_ALGORITHMS.includes(jwtAlg);
+    } else {
+      // We don't support 'ec' yet
+      return false;
+    }
+  }
+
+  async isValidSignature(token: string): Promise<boolean> {
+    try {
+      await jose.compactVerify(token, this.key);
+      return true;
+    } catch (e) {
+      if (e.code == 'ERR_JWS_SIGNATURE_VERIFICATION_FAILED') {
+        return false;
+      } else {
+        // Token format error most likely
+        throw e;
+      }
+    }
+  }
+}