@powersync/service-core 0.0.0-dev-20240708120322 → 0.0.0-dev-20240718134716

package/src/util/config/compound-config-collector.ts

@@ -1,3 +1,4 @@
+import * as t from 'ts-codec';
 import { configFile, normalizeConnection } from '@powersync/service-types';
 import { ConfigCollector } from './collectors/config-collector.js';
 import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
@@ -9,7 +10,7 @@ import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-co
 import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
 import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
 import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
-import { logger } from '@powersync/lib-services-framework';
+import { logger, schema } from '@powersync/lib-services-framework';
 
 const POWERSYNC_DEV_KID = 'powersync-dev';
 
@@ -28,6 +29,12 @@ export type CompoundConfigCollectorOptions = {
   syncRulesCollectors: SyncRulesCollector[];
 };
 
+export type ConfigCollectorGenerics = {
+  SERIALIZED: configFile.SerializedPowerSyncConfig;
+  DESERIALIZED: configFile.PowerSyncConfig;
+  RESOLVED: ResolvedPowerSyncConfig;
+};
+
 const DEFAULT_COLLECTOR_OPTIONS: CompoundConfigCollectorOptions = {
   configCollectors: [new Base64ConfigCollector(), new FileSystemConfigCollector(), new FallbackConfigCollector()],
   syncRulesCollectors: [
@@ -37,15 +44,56 @@ const DEFAULT_COLLECTOR_OPTIONS: CompoundConfigCollectorOptions = {
   ]
 };
 
-export class CompoundConfigCollector {
+export class CompoundConfigCollector<Generics extends ConfigCollectorGenerics = ConfigCollectorGenerics> {
   constructor(protected options: CompoundConfigCollectorOptions = DEFAULT_COLLECTOR_OPTIONS) {}
 
+  /**
+   * The default ts-codec for validations and decoding
+   */
+  get codec(): t.AnyCodec {
+    return configFile.powerSyncConfig;
+  }
+
   /**
    * Collects and resolves base config
    */
-  async collectConfig(runner_config: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
-    const baseConfig = await this.collectBaseConfig(runner_config);
+  async collectConfig(runnerConfig: RunnerConfig = {}): Promise<Generics['RESOLVED']> {
+    const baseConfig = await this.collectBaseConfig(runnerConfig);
+    const baseResolvedConfig = await this.resolveBaseConfig(baseConfig, runnerConfig);
+    return this.resolveConfig(baseConfig, baseResolvedConfig, runnerConfig);
+  }
+
+  /**
+   * Collects the base PowerSyncConfig from various registered collectors.
+   * @throws if no collector could return a configuration.
+   */
+  protected async collectBaseConfig(runner_config: RunnerConfig): Promise<Generics['DESERIALIZED']> {
+    for (const collector of this.options.configCollectors) {
+      try {
+        const baseConfig = await collector.collectSerialized(runner_config);
+        if (baseConfig) {
+          const decoded = this.decode(baseConfig);
+          this.validate(decoded);
+          return decoded;
+        }
+        logger.debug(
+          `Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`
+        );
+      } catch (ex) {
+        // An error in a collector is a hard stop
+        throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
+      }
+    }
+    throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
+  }
 
+  /**
+   * Performs the resolving of the common (shared) base configuration
+   */
+  protected async resolveBaseConfig(
+    baseConfig: Generics['DESERIALIZED'],
+    runnerConfig: RunnerConfig = {}
+  ): Promise<ResolvedPowerSyncConfig> {
     const connections = baseConfig.replication?.connections ?? [];
     if (connections.length > 1) {
       throw new Error('Only a single replication connection is supported currently');
@@ -93,7 +141,7 @@ export class CompoundConfigCollector {
       devKey = await auth.KeySpec.importKey(baseDevKey);
     }
 
-    const sync_rules = await this.collectSyncRules(baseConfig, runner_config);
+    const sync_rules = await this.collectSyncRules(baseConfig, runnerConfig);
 
     let jwt_audiences: string[] = baseConfig.client_auth?.audience ?? [];
 
@@ -130,25 +178,17 @@ export class CompoundConfigCollector {
   }
 
   /**
-   * Collects the base PowerSyncConfig from various registered collectors.
-   * @throws if no collector could return a configuration.
+   * Perform any additional resolving from {@link ResolvedPowerSyncConfig}
+   * to the extended {@link Generics['RESOLVED']}
+   *
    */
-  protected async collectBaseConfig(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig> {
-    for (const collector of this.options.configCollectors) {
-      try {
-        const baseConfig = await collector.collect(runner_config);
-        if (baseConfig) {
-          return baseConfig;
-        }
-        logger.debug(
-          `Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`
-        );
-      } catch (ex) {
-        // An error in a collector is a hard stop
-        throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
-      }
-    }
-    throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
+  protected async resolveConfig(
+    baseConfig: Generics['DESERIALIZED'],
+    resolvedBaseConfig: ResolvedPowerSyncConfig,
+    runnerConfig: RunnerConfig = {}
+  ): Promise<Generics['RESOLVED']> {
+    // The base version has ResolvedPowerSyncConfig == Generics['RESOLVED']
+    return resolvedBaseConfig;
   }
 
   protected async collectSyncRules(
@@ -173,4 +213,28 @@ export class CompoundConfigCollector {
       present: false
     };
   }
+
+  /**
+   * Validates input config
+   * ts-codec itself doesn't give great validation errors, so we use json schema for that
+   */
+  protected validate(config: Generics['DESERIALIZED']) {
+    // ts-codec itself doesn't give great validation errors, so we use json schema for that
+    const validator = schema
+      .parseJSONSchema(t.generateJSONSchema(this.codec, { allowAdditional: true, parsers: [configFile.portParser] }))
+      .validator();
+
+    const valid = validator.validate(config);
+    if (!valid.valid) {
+      throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
+    }
+  }
+
+  protected decode(encoded: Generics['SERIALIZED']): Generics['DESERIALIZED'] {
+    try {
+      return this.codec.decode(encoded);
+    } catch (ex) {
+      throw new Error(`Failed to decode PowerSync config: ${ex}`);
+    }
+  }
 }

package/test/src/sync.test.ts

@@ -9,6 +9,7 @@ import { streamResponse } from '../../src/sync/sync.js';
 import * as timers from 'timers/promises';
 import { lsnMakeComparable } from '@powersync/service-jpgwire';
 import { RequestParameters } from '@powersync/service-sync-rules';
+import { RequestTracker } from '@/sync/RequestTracker.js';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
@@ -38,6 +39,8 @@ bucket_definitions:
     `;
 
 function defineTests(factory: StorageFactory) {
+  const tracker = new RequestTracker();
+
   test('sync global data', async () => {
     const f = await factory();
 
@@ -78,6 +81,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
     });
@@ -118,6 +122,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: false
       },
+      tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
     });
@@ -146,6 +151,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
     });
@@ -172,6 +178,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
     });
@@ -232,6 +239,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
     });

package/test/src/util.ts

@@ -7,14 +7,20 @@ import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
 import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
 import { env } from './env.js';
 import { Metrics } from '@/metrics/Metrics.js';
+import { container } from '@powersync/lib-services-framework';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 
 // The metrics need to be initialised before they can be used
-await Metrics.initialise({
-  disable_telemetry_sharing: true,
-  powersync_instance_id: 'test',
-  internal_metrics_endpoint: 'unused.for.tests.com'
-});
-Metrics.getInstance().resetCounters();
+const prometheus = new PrometheusExporter();
+const metrics = new Metrics(
+  new MeterProvider({
+    readers: [prometheus]
+  }),
+  prometheus
+);
+container.register(Metrics, metrics);
+metrics.resetCounters();
 
 export const TEST_URI = env.PG_TEST_URL;
 

package/test/src/wal_stream.test.ts

@@ -5,6 +5,7 @@ import { MONGO_STORAGE_FACTORY } from './util.js';
 import { putOp, removeOp, walStreamTest } from './wal_stream_utils.js';
 import { pgwireRows } from '@powersync/service-jpgwire';
 import { Metrics } from '@/metrics/Metrics.js';
+import { container } from '@powersync/lib-services-framework';
 
 type StorageFactory = () => Promise<BucketStorageFactory>;
 
@@ -41,10 +42,9 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const startRowCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const metrics = container.getImplementation(Metrics);
+      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -59,9 +59,8 @@ bucket_definitions:
       expect(data).toMatchObject([
         putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })
       ]);
-      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
       expect(endRowCount - startRowCount).toEqual(1);
       expect(endTxCount - startTxCount).toEqual(1);
     })
@@ -83,10 +82,9 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const startRowCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const metrics = container.getImplementation(Metrics);
+      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -97,9 +95,8 @@ bucket_definitions:
       const data = await context.getBucketData('global[]');
 
       expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
-      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
       expect(endRowCount - startRowCount).toEqual(1);
       expect(endTxCount - startTxCount).toEqual(1);
     })
@@ -293,10 +290,9 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const startRowCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const metrics = container.getImplementation(Metrics);
+      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -307,9 +303,8 @@ bucket_definitions:
       const data = await context.getBucketData('global[]');
 
       expect(data).toMatchObject([]);
-      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       // There was a transaction, but we should not replicate any actual data
       expect(endRowCount - startRowCount).toEqual(0);