@powersync/service-core 0.0.2

package/src/util/config/collectors/config-collector.ts

@@ -0,0 +1,141 @@
+import * as t from 'ts-codec';
+import { configFile } from '@powersync/service-types';
+import * as micro from '@journeyapps-platform/micro';
+import { RunnerConfig } from '../types.js';
+import * as yaml from 'yaml';
+
+export enum ConfigFileFormat {
+  YAML = 'yaml',
+  JSON = 'json'
+}
+
+/**
+ * Environment variables can be substituted into the YAML config
+ * when parsing if the environment variable name starts with this prefix.
+ * Attempting to substitute any other environment variable will throw an exception.
+ *
+ * Example of substitution:
+ * storage:
+ *    type: mongodb
+ *    uri: !env PS_MONGO_URI
+ */
+const YAML_ENV_PREFIX = 'PS_';
+
+// ts-codec itself doesn't give great validation errors, so we use json schema for that
+const configSchemaValidator = micro.schema
+  .parseJSONSchema(
+    t.generateJSONSchema(configFile.powerSyncConfig, { allowAdditional: true, parsers: [configFile.portParser] })
+  )
+  .validator();
+
+export abstract class ConfigCollector {
+  abstract get name(): string;
+
+  /**
+   * Collects the serialized base PowerSyncConfig.
+   * @returns null if this collector cannot provide a config
+   */
+  abstract collectSerialized(runnerConfig: RunnerConfig): Promise<configFile.SerializedPowerSyncConfig | null>;
+
+  /**
+   * Collects the PowerSyncConfig settings.
+   * Validates and decodes the config.
+   * @returns null if this collector cannot provide a config
+   */
+  async collect(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig | null> {
+    const serialized = await this.collectSerialized(runner_config);
+    if (!serialized) {
+      return null;
+    }
+
+    /**
+     * After this point a serialized config has been found. Any failures to decode or validate
+     * will result in a hard stop.
+     */
+    const decoded = this.decode(serialized);
+    this.validate(decoded);
+    return decoded;
+  }
+
+  /**
+   * Validates input config
+   * ts-codec itself doesn't give great validation errors, so we use json schema for that
+   */
+  validate(config: configFile.PowerSyncConfig) {
+    const valid = configSchemaValidator.validate(config);
+    if (!valid.valid) {
+      throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
+    }
+  }
+
+  decode(encoded: configFile.SerializedPowerSyncConfig): configFile.PowerSyncConfig {
+    try {
+      return configFile.powerSyncConfig.decode(encoded);
+    } catch (ex) {
+      throw new Error(`Failed to decode PowerSync config: ${ex}`);
+    }
+  }
+
+  protected parseContent(content: string, contentType?: ConfigFileFormat) {
+    switch (contentType) {
+      case ConfigFileFormat.YAML:
+        return this.parseYaml(content);
+      case ConfigFileFormat.JSON:
+        return this.parseJSON(content);
+      default: {
+        // No content type provided, need to try both
+        try {
+          return this.parseYaml(content);
+        } catch (ex) {}
+        try {
+          return this.parseJSON(content);
+        } catch (ex) {
+          throw new Error(`Could not parse PowerSync config file content as JSON or YAML: ${ex}`);
+        }
+      }
+    }
+  }
+
+  protected parseYaml(content: string) {
+    const lineCounter = new yaml.LineCounter();
+
+    const parsed = yaml.parseDocument(content, {
+      schema: 'core',
+      keepSourceTokens: true,
+      lineCounter,
+      customTags: [
+        {
+          tag: '!env',
+          resolve(envName: string, onError: (error: string) => void) {
+            if (!envName.startsWith(YAML_ENV_PREFIX)) {
+              onError(
+                `Attempting to substitute environment variable ${envName} is not allowed. Variables must start with "${YAML_ENV_PREFIX}"`
+              );
+              return envName;
+            }
+            const value = process.env[envName];
+            if (typeof value == 'undefined') {
+              onError(
+                `Attempted to substitute environment variable "${envName}" which is undefined. Set this variable on the environment.`
+              );
+              return envName;
+            }
+            return value;
+          }
+        }
+      ]
+    });
+
+    if (parsed.errors.length) {
+      throw new Error(
+        `Could not parse YAML configuration file. Received errors: \n ${parsed.errors.map((e) => e.message).join('\n')}`
+      );
+    }
+
+    return parsed.toJS();
+  }
+
+  protected parseJSON(content: string) {
+    return JSON.parse(content);
+  }
+}

package/src/util/config/collectors/impl/base64-config-collector.ts

@@ -0,0 +1,18 @@
+import { ConfigCollector } from '../config-collector.js';
+import { RunnerConfig } from '../../types.js';
+
+export class Base64ConfigCollector extends ConfigCollector {
+  get name(): string {
+    return 'Base64';
+  }
+
+  async collectSerialized(runnerConfig: RunnerConfig) {
+    const { config_base64 } = runnerConfig;
+    if (!config_base64) {
+      return null;
+    }
+
+    // Could be JSON or YAML at this point
+    return this.parseContent(Buffer.from(config_base64, 'base64').toString());
+  }
+}

package/src/util/config/collectors/impl/fallback-config-collector.ts

@@ -0,0 +1,22 @@
+import { RunnerConfig } from '../../types.js';
+import { FileSystemConfigCollector } from './filesystem-config-collector.js';
+
+export const DEFAULT_CONFIG_LOCATION = 'powersync.yaml';
+
+/**
+ * Falls back to reading the PowerSync config file from the previous
+ * default location of `powersync.yaml`
+ */
+export class FallbackConfigCollector extends FileSystemConfigCollector {
+  get name(): string {
+    return 'Fallback powersync.yaml';
+  }
+
+  async collectSerialized(runnerConfig: RunnerConfig) {
+    return super.collectSerialized({
+      ...runnerConfig,
+      // Use the fallback config location
+      config_path: DEFAULT_CONFIG_LOCATION
+    });
+  }
+}

package/src/util/config/collectors/impl/filesystem-config-collector.ts

@@ -0,0 +1,41 @@
+import * as fs from 'fs/promises';
+import * as micro from '@journeyapps-platform/micro';
+
+import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
+import { RunnerConfig } from '../../types.js';
+
+export class FileSystemConfigCollector extends ConfigCollector {
+  get name(): string {
+    return 'FileSystem';
+  }
+
+  async collectSerialized(runnerConfig: RunnerConfig) {
+    const { config_path } = runnerConfig;
+    if (!config_path) {
+      return null;
+    }
+
+    // Check if file exists
+    try {
+      await fs.access(config_path, fs.constants.F_OK);
+    } catch (ex) {
+      throw new Error(`Config file path ${config_path} was specified, but the file does not exist.`);
+    }
+
+    micro.logger.info(`Collecting PowerSync configuration from File: ${config_path}`);
+
+    const content = await fs.readFile(config_path, 'utf-8');
+
+    let contentType: ConfigFileFormat | undefined;
+    switch (true) {
+      case config_path.endsWith('.yaml'):
+      case config_path.endsWith('.yml'):
+        contentType = ConfigFileFormat.YAML;
+        break;
+      case config_path.endsWith('.json'):
+        contentType = ConfigFileFormat.JSON;
+        break;
+    }
+    return this.parseContent(content, contentType);
+  }
+}

package/src/util/config/compound-config-collector.ts

@@ -0,0 +1,171 @@
+import * as micro from '@journeyapps-platform/micro';
+import { configFile, normalizeConnection } from '@powersync/service-types';
+import { ConfigCollector } from './collectors/config-collector.js';
+import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
+import * as auth from '@/auth/auth-index.js';
+import { SyncRulesCollector } from './sync-rules/sync-collector.js';
+import { Base64ConfigCollector } from './collectors/impl/base64-config-collector.js';
+import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
+import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-collector.js';
+import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
+import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
+import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
+
+const POWERSYNC_DEV_KID = 'powersync-dev';
+
+export type CompoundConfigCollectorOptions = {
+  /**
+   * Collectors for PowerSync configuration content.
+   * The configuration from first collector to provide a configuration
+   * is used. The order of the collectors specifies precedence
+   */
+  configCollectors: ConfigCollector[];
+  /**
+   * Collectors for PowerSync sync rules content.
+   * The configuration from first collector to provide a configuration
+   * is used. The order of the collectors specifies precedence
+   */
+  syncRulesCollectors: SyncRulesCollector[];
+};
+
+const DEFAULT_COLLECTOR_OPTIONS: CompoundConfigCollectorOptions = {
+  configCollectors: [new Base64ConfigCollector(), new FileSystemConfigCollector(), new FallbackConfigCollector()],
+  syncRulesCollectors: [
+    new Base64SyncRulesCollector(),
+    new FileSystemSyncRulesCollector(),
+    new InlineSyncRulesCollector()
+  ]
+};
+
+export class CompoundConfigCollector {
+  constructor(protected options: CompoundConfigCollectorOptions = DEFAULT_COLLECTOR_OPTIONS) {}
+
+  /**
+   * Collects and resolves base config
+   */
+  async collectConfig(runner_config: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
+    const baseConfig = await this.collectBaseConfig(runner_config);
+
+    const connections = baseConfig.replication?.connections ?? [];
+    if (connections.length > 1) {
+      throw new Error('Only a single replication connection is supported currently');
+    }
+
+    const mapped = connections.map((c) => {
+      const conf: ResolvedConnection = {
+        type: 'postgresql' as const,
+        ...normalizeConnection(c),
+        debug_api: c.debug_api ?? false
+      };
+
+      return conf;
+    });
+
+    const collectors = new auth.CompoundKeyCollector();
+    const keyStore = new auth.KeyStore(collectors);
+
+    const inputKeys = baseConfig.client_auth?.jwks?.keys ?? [];
+    const staticCollector = await auth.StaticKeyCollector.importKeys(inputKeys);
+
+    collectors.add(staticCollector);
+
+    if (baseConfig.client_auth?.supabase && mapped.length > 0) {
+      collectors.add(new auth.CachedKeyCollector(new auth.SupabaseKeyCollector(mapped[0])));
+    }
+
+    let jwks_uris = baseConfig.client_auth?.jwks_uri ?? [];
+    if (typeof jwks_uris == 'string') {
+      jwks_uris = [jwks_uris];
+    }
+
+    for (let uri of jwks_uris) {
+      collectors.add(
+        new auth.CachedKeyCollector(
+          new auth.RemoteJWKSCollector(uri, { block_local_ip: !!baseConfig.client_auth?.block_local_jwks })
+        )
+      );
+    }
+
+    const baseDevKey = (baseConfig.client_auth?.jwks?.keys ?? []).find((key) => key.kid == POWERSYNC_DEV_KID);
+
+    let devKey: auth.KeySpec | undefined;
+    if (baseConfig.dev?.demo_auth && baseDevKey != null && baseDevKey.kty == 'oct') {
+      devKey = await auth.KeySpec.importKey(baseDevKey);
+    }
+
+    const sync_rules = await this.collectSyncRules(baseConfig, runner_config);
+
+    let jwt_audiences: string[] = baseConfig.client_auth?.audience ?? [];
+
+    let config: ResolvedPowerSyncConfig = {
+      connection: mapped[0],
+      storage: baseConfig.storage,
+      client_keystore: keyStore,
+      // Dev tokens only use the static keys, no external key sources
+      // We may restrict this even further to only the powersync-dev key.
+      dev_client_keystore: new auth.KeyStore(staticCollector),
+      api_tokens: baseConfig.api?.tokens ?? [],
+      dev: {
+        demo_auth: baseConfig.dev?.demo_auth ?? false,
+        demo_client: baseConfig.dev?.demo_client ?? false,
+        demo_password: baseConfig.dev?.demo_password,
+        crud_api: baseConfig.dev?.crud_api ?? false,
+        dev_key: devKey
+      },
+      port: baseConfig.port ?? 8080,
+      sync_rules,
+      jwt_audiences,
+
+      token_max_expiration: '1d', // 1 day
+      metadata: baseConfig.metadata ?? {},
+      migrations: baseConfig.migrations,
+      slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
+    };
+    return config;
+  }
+
+  /**
+   * Collects the base PowerSyncConfig from various registered collectors.
+   * @throws if no collector could return a configuration.
+   */
+  protected async collectBaseConfig(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig> {
+    for (const collector of this.options.configCollectors) {
+      try {
+        const baseConfig = await collector.collect(runner_config);
+        if (baseConfig) {
+          return baseConfig;
+        }
+        micro.logger.debug(
+          `Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`
+        );
+      } catch (ex) {
+        // An error in a collector is a hard stop
+        throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
+      }
+    }
+    throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
+  }
+
+  protected async collectSyncRules(
+    baseConfig: configFile.PowerSyncConfig,
+    runnerConfig: RunnerConfig
+  ): Promise<SyncRulesConfig> {
+    for (const collector of this.options.syncRulesCollectors) {
+      try {
+        const config = await collector.collect(baseConfig, runnerConfig);
+        if (config) {
+          return config;
+        }
+        micro.logger.debug(
+          `Could not collect sync rules with ${collector.name} method. Moving on to next method if available.`
+        );
+      } catch (ex) {
+        // An error in a collector is a hard stop
+        throw new Error(`Could not collect sync rules using ${collector.name} method. Caught exception: ${ex}`);
+      }
+    }
+    return {
+      present: false
+    };
+  }
+}

package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts

@@ -0,0 +1,21 @@
+import { RunnerConfig, SyncRulesConfig } from '../../types.js';
+import { SyncRulesCollector } from '../sync-collector.js';
+import { configFile } from '@powersync/service-types';
+
+export class Base64SyncRulesCollector extends SyncRulesCollector {
+  get name(): string {
+    return 'Base64';
+  }
+
+  async collect(baseConfig: configFile.PowerSyncConfig, runnerConfig: RunnerConfig): Promise<SyncRulesConfig | null> {
+    const { sync_rules_base64 } = runnerConfig;
+    if (!sync_rules_base64) {
+      return null;
+    }
+
+    return {
+      present: true,
+      content: Buffer.from(sync_rules_base64, 'base64').toString()
+    };
+  }
+}

package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts

@@ -0,0 +1,26 @@
+import * as path from 'path';
+import { RunnerConfig, SyncRulesConfig } from '../../types.js';
+import { SyncRulesCollector } from '../sync-collector.js';
+import { configFile } from '@powersync/service-types';
+
+export class FileSystemSyncRulesCollector extends SyncRulesCollector {
+  get name(): string {
+    return 'FileSystem';
+  }
+
+  async collect(baseConfig: configFile.PowerSyncConfig, runnerConfig: RunnerConfig): Promise<SyncRulesConfig | null> {
+    const sync_path = baseConfig.sync_rules?.path;
+    if (!sync_path) {
+      return null;
+    }
+
+    const { config_path } = runnerConfig;
+
+    // Depending on the container, the sync rules may not actually be present.
+    // Only persist the path here, and load on demand using `loadSyncRules()`.
+    return {
+      present: true,
+      path: config_path ? path.resolve(path.dirname(config_path), sync_path) : sync_path
+    };
+  }
+}

package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts

@@ -0,0 +1,21 @@
+import { SyncRulesConfig } from '../../types.js';
+import { SyncRulesCollector } from '../sync-collector.js';
+import { configFile } from '@powersync/service-types';
+
+export class InlineSyncRulesCollector extends SyncRulesCollector {
+  get name(): string {
+    return 'Inline';
+  }
+
+  async collect(baseConfig: configFile.PowerSyncConfig): Promise<SyncRulesConfig | null> {
+    const content = baseConfig.sync_rules?.content;
+    if (!content) {
+      return null;
+    }
+
+    return {
+      present: true,
+      ...baseConfig.sync_rules
+    };
+  }
+}

package/src/util/config/sync-rules/sync-collector.ts

@@ -0,0 +1,8 @@
+import { configFile } from '@powersync/service-types';
+import { RunnerConfig, SyncRulesConfig } from '../types.js';
+
+export abstract class SyncRulesCollector {
+  abstract get name(): string;
+
+  abstract collect(baseConfig: configFile.PowerSyncConfig, runnerConfig: RunnerConfig): Promise<SyncRulesConfig | null>;
+}

package/src/util/config/types.ts

@@ -0,0 +1,60 @@
+import { NormalizedPostgresConnection, configFile } from '@powersync/service-types';
+import { KeySpec } from '../../auth/KeySpec.js';
+import { KeyStore } from '../../auth/KeyStore.js';
+
+export enum ServiceRunner {
+  UNIFIED = 'unified',
+  API = 'api',
+  SYNC = 'sync'
+}
+
+export type RunnerConfig = {
+  config_path?: string;
+  config_base64?: string;
+  sync_rules_base64?: string;
+};
+
+export type MigrationContext = {
+  runner_config: RunnerConfig;
+};
+
+export type Runner = (config: RunnerConfig) => Promise<void>;
+
+export type ResolvedConnection = configFile.PostgresConnection & NormalizedPostgresConnection;
+
+export type SyncRulesConfig = {
+  present: boolean;
+  content?: string;
+  path?: string;
+};
+
+export type ResolvedPowerSyncConfig = {
+  connection?: ResolvedConnection;
+  storage: configFile.StorageConfig;
+  dev: {
+    demo_auth: boolean;
+    demo_password?: string;
+    crud_api: boolean;
+    demo_client: boolean;
+    /**
+     * Only present when demo_auth == true
+     */
+    dev_key?: KeySpec;
+  };
+  client_keystore: KeyStore;
+  /**
+   * Keystore for development tokens.
+   */
+  dev_client_keystore: KeyStore;
+  port: number;
+  sync_rules: SyncRulesConfig;
+  api_tokens: string[];
+  jwt_audiences: string[];
+  token_max_expiration: string;
+  metadata: Record<string, string>;
+  migrations?: {
+    disable_auto_migration?: boolean;
+  };
+  /** Prefix for postgres replication slot names. May eventually be connection-specific. */
+  slot_name_prefix: string;
+};

package/src/util/config.ts

@@ -0,0 +1,39 @@
+import * as fs from 'fs/promises';
+import { baseUri } from '@powersync/service-types';
+
+import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
+import { CompoundConfigCollector } from './config/compound-config-collector.js';
+
+/**
+ * Build a single URI from full postgres credentials.
+ */
+export function buildDemoPgUri(options: ResolvedConnection): string {
+  if (!options.debug_api) {
+    throw new Error('Not supported');
+  }
+
+  const uri = new URL(baseUri(options));
+  uri.username = options.username;
+  uri.password = options.password;
+  if (options.sslmode != 'disable') {
+    // verify-full is tricky to actually use on a client, since they won't have the cert
+    // Just use "require" by default
+    // uri.searchParams.set('sslmode', options.sslmode);
+    uri.searchParams.set('sslmode', 'require');
+  }
+  return uri.toString();
+}
+
+export function loadConfig(runnerConfig: RunnerConfig = {}) {
+  const collector = new CompoundConfigCollector();
+  return collector.collectConfig(runnerConfig);
+}
+
+export async function loadSyncRules(config: ResolvedPowerSyncConfig): Promise<string | undefined> {
+  const sync_rules = config.sync_rules;
+  if (sync_rules.content) {
+    return sync_rules.content;
+  } else if (sync_rules.path) {
+    return await fs.readFile(sync_rules.path, 'utf-8');
+  }
+}

package/src/util/env.ts

@@ -0,0 +1,28 @@
+import { utils } from '@journeyapps-platform/micro';
+
+import { ServiceRunner } from './config/types.js';
+
+export const env = utils.collectEnvironmentVariables({
+  /**
+   * Path to configuration file in filesystem
+   */
+  POWERSYNC_CONFIG_PATH: utils.type.string.optional(),
+  /**
+   * Base64 encoded contents of configuration file
+   */
+  POWERSYNC_CONFIG_B64: utils.type.string.optional(),
+  /**
+   * Base64 encoded contents of sync rules YAML
+   */
+  POWERSYNC_SYNC_RULES_B64: utils.type.string.optional(),
+  /**
+   * Runner to be started in this process
+   */
+  PS_RUNNER_TYPE: utils.type.string.default(ServiceRunner.UNIFIED),
+  /**
+   * Port for metrics
+   */
+  METRICS_PORT: utils.type.number.optional()
+});
+
+export type Env = typeof env;

package/src/util/memory-tracking.ts

@@ -0,0 +1,67 @@
+import * as micro from '@journeyapps-platform/micro';
+
+/**
+ * Track and log memory usage.
+ *
+ * Only for debugging purposes. This could add significant overhead and/or side-effects,
+ * and should not be used in production.
+ */
+export function trackMemoryUsage() {
+  let lastMem = process.memoryUsage();
+
+  let bufferMemory = {
+    alloc: 0,
+    allocUnsafe: 0,
+    allocUnsafeSlow: 0,
+    from: 0,
+    concat: 0
+  };
+
+  const bufferRegistry = new FinalizationRegistry<[keyof typeof bufferMemory, number]>((v) => {
+    const [key, value] = v;
+    bufferMemory[key] -= value;
+  });
+
+  for (let key of Object.keys(bufferMemory)) {
+    const typedKey = key as keyof typeof bufferMemory;
+    const originalFunction = Buffer[typedKey] as (...args: any[]) => Buffer;
+    Buffer[typedKey] = function (...args: any[]) {
+      const buffer = originalFunction.apply(this, args);
+      bufferMemory[typedKey] += buffer.byteLength;
+      bufferRegistry.register(buffer, [typedKey, buffer.byteLength]);
+      return buffer;
+    };
+  }
+
+  setInterval(() => {
+    const mem = process.memoryUsage();
+
+    let isDifferent = false;
+    for (const key in mem) {
+      if (Math.abs((mem as any)[key] - (lastMem as any)[key]) > 5_000_000) {
+        isDifferent = true;
+      }
+    }
+
+    if (isDifferent) {
+      lastMem = mem;
+
+      const output = `RSS ${mb(mem.rss)} (
+        HAT ${mb(mem.heapTotal)} (
+          HU ${mb(mem.heapUsed)}
+        ),
+        buffers ${mb(mem.arrayBuffers)} (
+          alloc ${mb(bufferMemory.alloc + bufferMemory.allocUnsafe + bufferMemory.allocUnsafeSlow)},
+          from ${mb(bufferMemory.from)}
+          concat ${mb(bufferMemory.concat)}
+        )
+      )`.replaceAll(/\s+/g, ' ');
+
+      micro.logger.info(output);
+    }
+  }, 50);
+}
+
+function mb(n: number) {
+  return Math.round(n / 1024 / 1024) + 'mb';
+}