@powersync/service-core 0.0.0-dev-20260313100403 → 0.0.0-dev-20260515144844

package/src/util/config/collectors/impl/filesystem-config-collector.ts

@@ -1,9 +1,9 @@
 import * as fs from 'fs/promises';
 import * as path from 'path';
 
-import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
-import { RunnerConfig } from '../../types.js';
 import { logger } from '@powersync/lib-services-framework';
+import { RunnerConfig } from '../../types.js';
+import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
 
 export class FileSystemConfigCollector extends ConfigCollector {
   get name(): string {

package/src/util/config/compound-config-collector.ts

@@ -26,7 +26,7 @@ export type CompoundConfigCollectorOptions = {
    */
   configCollectors: ConfigCollector[];
   /**
-   * Collectors for PowerSync sync rules content.
+   * Collectors for PowerSync sync config content.
    * The configuration from first collector to provide a configuration
    * is used. The order of the collectors specifies precedence
    */
@@ -236,11 +236,11 @@ export class CompoundConfigCollector {
           return config;
         }
         logger.debug(
-          `Could not collect sync rules with ${collector.name} method. Moving on to next method if available.`
+          `Could not collect sync config with ${collector.name} method. Moving on to next method if available.`
         );
       } catch (ex) {
         // An error in a collector is a hard stop
-        throw new Error(`Could not collect sync rules using ${collector.name} method. Caught exception: ${ex}`);
+        throw new Error(`Could not collect sync config using ${collector.name} method. Caught exception: ${ex}`);
       }
     }
     return {

package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts

@@ -16,7 +16,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
 
     const { config_path } = runnerConfig;
 
-    // Depending on the container, the sync rules may not actually be present.
+    // Depending on the container, the sync config may not actually be present.
     // Only persist the path here, and load on demand using `loadSyncRules()`.
     return {
       present: true,

package/src/util/config/sync-rules/sync-rules-provider.ts

@@ -1,5 +1,5 @@
-import { SyncRulesConfig } from '../types.js';
 import fs from 'fs/promises';
+import { SyncRulesConfig } from '../types.js';
 
 export interface SyncRulesProvider {
   get(): Promise<string | undefined>;

package/src/util/config.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs/promises';
 import winston from 'winston';
 
-import { container, logger, LogFormat, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT } from '@powersync/lib-services-framework';
+import { container, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, LogFormat, logger } from '@powersync/lib-services-framework';
 import { configFile } from '@powersync/service-types';
 import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
 import { CompoundConfigCollector } from './util-index.js';

package/src/util/env.ts

@@ -13,7 +13,7 @@ export const env = utils.collectEnvironmentVariables({
   POWERSYNC_CONFIG_B64: utils.type.string.optional(),
   /**
    * @deprecated use POWERSYNC_SYNC_CONFIG_B64 instead.
-   * Base64 encoded contents of sync rules YAML
+   * Base64 encoded contents of sync config YAML
    */
   POWERSYNC_SYNC_RULES_B64: utils.type.string.optional(),
   /**

package/src/util/errors.ts

@@ -0,0 +1,21 @@
+import { YamlError } from '@powersync/service-sync-rules';
+import { ReplicationError } from '@powersync/service-types';
+
+export function syncConfigYamlErrorToReplicationError(
+  { type, message, location }: YamlError,
+  ts?: string
+): ReplicationError {
+  const error: ReplicationError = {
+    level: type,
+    message,
+    ts
+  };
+  if (location != null) {
+    error.location = {
+      start_offset: location.start,
+      end_offset: location.end
+    };
+  }
+
+  return error;
+}

package/src/util/protocol-types.ts

@@ -77,7 +77,7 @@ export const StreamingSyncRequest = t.object({
   raw_data: t.boolean.optional(),
 
   /**
-   * Client parameters to be passed to the sync rules.
+   * Client parameters to be passed to the sync config.
    */
   parameters: t.record(t.any).optional(),
 

package/src/util/util-index.ts

@@ -1,4 +1,5 @@
 export * from './alerting.js';
+export * from './checkpointing.js';
 export * from './env.js';
 export * from './lsn.js';
 export * from './memory-tracking.js';
@@ -6,7 +7,6 @@ export * from './Mutex.js';
 export * from './protocol-types.js';
 export * from './secs.js';
 export * from './utils.js';
-export * from './checkpointing.js';
 export * from './version.js';
 
 export * from './config.js';

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
+import { BucketChecksum, OplogEntry, ProtocolOpId } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 

package/test/src/ReplicationLagTracker.test.ts

@@ -0,0 +1,53 @@
+import { ReplicationLagTracker } from '@/replication/ReplicationLagTracker.js';
+import { describe, expect, it } from 'vitest';
+
+describe('ReplicationLagTracker', () => {
+  it('returns undefined before replication has started', () => {
+    const tracker = new ReplicationLagTracker();
+
+    expect(tracker.getLagMillis(0)).toBeUndefined();
+  });
+
+  it('tracks the oldest in-flight change and returns current lag', () => {
+    const tracker = new ReplicationLagTracker();
+
+    tracker.trackUncommittedChange(new Date(1_000));
+    tracker.trackUncommittedChange(new Date(2_000));
+
+    expect(tracker.oldestUncommittedChange?.getTime()).toBe(1_000);
+    expect(tracker.getCurrentLagMillis(4_000)).toBe(3_000);
+    expect(tracker.getLagMillis(4_000)).toBe(3_000);
+  });
+
+  it('records commit lag into the rolling window and clears in-flight state', () => {
+    const tracker = new ReplicationLagTracker();
+
+    tracker.trackUncommittedChange(new Date(0));
+    tracker.markCommitted(5_000);
+
+    expect(tracker.oldestUncommittedChange).toBeNull();
+    expect(tracker.isStartingReplication).toBe(false);
+    expect(tracker.getLagMillis(5_000)).toBe(5_000);
+    expect(tracker.getLagMillis(35_000)).toBe(0);
+  });
+
+  it('can clear in-flight state without changing started state', () => {
+    const tracker = new ReplicationLagTracker();
+
+    tracker.trackUncommittedChange(new Date(0));
+    tracker.clearUncommittedChange();
+
+    expect(tracker.oldestUncommittedChange).toBeNull();
+    expect(tracker.isStartingReplication).toBe(true);
+    expect(tracker.getLagMillis(5_000)).toBeUndefined();
+  });
+
+  it('can mark replication as started without a committed transaction', () => {
+    const tracker = new ReplicationLagTracker();
+
+    tracker.markStarted();
+
+    expect(tracker.isStartingReplication).toBe(false);
+    expect(tracker.getLagMillis(0)).toBe(0);
+  });
+});

package/test/src/RollingBucketMax.test.ts

@@ -0,0 +1,106 @@
+import { RollingBucketMax } from '@/metrics/RollingBucketMax.js';
+import { describe, expect, it } from 'vitest';
+
+describe('RollingBucketMax', () => {
+  it('returns undefined before any values are reported', () => {
+    const tracker = new RollingBucketMax();
+
+    expect(tracker.getRollingMax(0)).toBeUndefined();
+  });
+
+  it('tracks the maximum value within a single bucket', () => {
+    const tracker = new RollingBucketMax();
+
+    tracker.report(3, 100);
+    tracker.report(9, 1_000);
+    tracker.report(5, 4_999);
+
+    expect(tracker.getRollingMax(4_999)).toBe(9);
+  });
+
+  it('keeps the rolling max across the last six 5s buckets', () => {
+    const tracker = new RollingBucketMax();
+
+    tracker.report(20, 0);
+    tracker.report(11, 5_000);
+    tracker.report(12, 10_000);
+    tracker.report(13, 15_000);
+    tracker.report(14, 20_000);
+    tracker.report(15, 25_000);
+
+    expect(tracker.getRollingMax(29_999)).toBe(20);
+    expect(tracker.getRollingMax(30_000)).toBe(15);
+  });
+
+  it('slides the rolling max forward as older buckets age out', () => {
+    const tracker = new RollingBucketMax();
+
+    tracker.report(20, 0);
+    expect(tracker.getRollingMax(0)).toBe(20);
+
+    tracker.report(18, 5_000);
+    expect(tracker.getRollingMax(5_000)).toBe(20);
+
+    tracker.report(16, 10_000);
+    expect(tracker.getRollingMax(10_000)).toBe(20);
+
+    tracker.report(14, 15_000);
+    expect(tracker.getRollingMax(15_000)).toBe(20);
+
+    tracker.report(12, 20_000);
+    expect(tracker.getRollingMax(20_000)).toBe(20);
+
+    tracker.report(10, 25_000);
+    expect(tracker.getRollingMax(29_999)).toBe(20);
+
+    tracker.report(8, 30_000);
+    expect(tracker.getRollingMax(30_000)).toBe(18);
+
+    tracker.report(6, 35_000);
+    expect(tracker.getRollingMax(35_000)).toBe(16);
+
+    tracker.report(4, 40_000);
+    expect(tracker.getRollingMax(40_000)).toBe(14);
+  });
+
+  it('keeps newer buckets in the rolling window while older peaks fall out', () => {
+    const tracker = new RollingBucketMax();
+
+    tracker.report(50, 0);
+    tracker.report(11, 5_000);
+    tracker.report(12, 10_000);
+    tracker.report(13, 15_000);
+    tracker.report(14, 20_000);
+    tracker.report(15, 25_000);
+
+    expect(tracker.getRollingMax(29_999)).toBe(50);
+
+    tracker.report(40, 30_000);
+    expect(tracker.getRollingMax(30_000)).toBe(40);
+    expect(tracker.getRollingMax(34_999)).toBe(40);
+  });
+
+  it('expires values after the rolling window passes with no new reports', () => {
+    const tracker = new RollingBucketMax();
+
+    tracker.report(7, 0);
+
+    expect(tracker.getRollingMax(29_999)).toBe(7);
+    expect(tracker.getRollingMax(30_000)).toBeUndefined();
+  });
+
+  it('supports custom bucket and window sizes', () => {
+    const tracker = new RollingBucketMax({
+      bucketSizeMs: 1_000,
+      windowSizeMs: 3_000
+    });
+
+    tracker.report(4, 0);
+    tracker.report(6, 1_000);
+    tracker.report(5, 2_000);
+
+    expect(tracker.getRollingMax(2_999)).toBe(6);
+    expect(tracker.getRollingMax(3_000)).toBe(6);
+    expect(tracker.getRollingMax(4_000)).toBe(5);
+  });
+});

package/test/src/auth.test.ts

@@ -1,13 +1,21 @@
-import { describe, expect, test } from 'vitest';
-import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import { StaticSupabaseKeyCollector } from '@/index.js';
+import { configFile } from '@powersync/service-types';
 import * as jose from 'jose';
+import { describe, expect, test } from 'vitest';
+import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
+import { KeyResult } from '../../src/auth/KeyCollector.js';
+import {
+  EC_ALGORITHMS,
+  HS_ALGORITHMS,
+  KeySpec,
+  OKP_ALGORITHMS,
+  RSA_ALGORITHMS,
+  SUPPORTED_ALGORITHMS
+} from '../../src/auth/KeySpec.js';
 import { KeyStore } from '../../src/auth/KeyStore.js';
-import { KeySpec } from '../../src/auth/KeySpec.js';
 import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
-import { KeyResult } from '../../src/auth/KeyCollector.js';
-import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
-import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
-import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
+import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import { getSupabaseJwksUrl } from '../../src/auth/utils.js';
 
 const publicKeyRSA: jose.JWK = {
   use: 'sig',
@@ -52,6 +60,47 @@ const privateKeyECDSA: jose.JWK = {
   alg: 'ES256'
 };
 
+const EC_ALGORITHM_CURVES = [
+  ['ES256', 'P-256'],
+  ['ES384', 'P-384'],
+  ['ES512', 'P-521']
+] satisfies [string, string][];
+
+const EDDSA_CURVES = ['Ed25519', 'Ed448'];
+
+function roundTripJwkThroughPowerSyncConfig(key: jose.JWK): jose.JWK {
+  const encoded = configFile.strictJwks.encode({
+    keys: [key as configFile.StrictJwk]
+  });
+
+  const decoded = configFile.strictJwks.decode(encoded);
+  return decoded.keys[0] as jose.JWK;
+}
+
+async function signAndVerifyWithKey(alg: string, key: jose.JWK, signKey: jose.KeyLike | Uint8Array) {
+  const parsedKey = roundTripJwkThroughPowerSyncConfig(key);
+  expect(parsedKey).toEqual(key);
+
+  const keys = await StaticKeyCollector.importKeys([parsedKey]);
+  const store = new KeyStore(keys);
+
+  const signedJwt = await new jose.SignJWT({ claim: alg })
+    .setProtectedHeader({ alg, kid: key.kid })
+    .setSubject('f1')
+    .setIssuedAt()
+    .setIssuer('tester')
+    .setAudience('tests')
+    .setExpirationTime('5m')
+    .sign(signKey);
+
+  const verified = await store.verifyJwt(signedJwt, {
+    defaultAudiences: ['tests'],
+    maxAge: '6m'
+  });
+
+  expect(verified.parsedPayload.claim).toEqual(alg);
+}
+
 describe('JWT Auth', () => {
   test('KeyStore basics', async () => {
     const keys = await StaticKeyCollector.importKeys([sharedKey]);
@@ -208,6 +257,65 @@ describe('JWT Auth', () => {
     ).rejects.toThrow('Unexpected token algorithm HS256');
   });
 
+  describe('supported JWT algorithms', () => {
+    test('covers every declared supported algorithm', () => {
+      const testedAlgorithms = new Set([
+        ...HS_ALGORITHMS,
+        ...RSA_ALGORITHMS,
+        ...EC_ALGORITHM_CURVES.map(([alg]) => alg),
+        ...OKP_ALGORITHMS
+      ]);
+
+      expect([...testedAlgorithms].sort()).toEqual([...SUPPORTED_ALGORITHMS].sort());
+    });
+
+    test.each(HS_ALGORITHMS)('verifies %s tokens', async (alg) => {
+      const secret = await jose.generateSecret(alg);
+      const key = await jose.exportJWK(secret);
+      key.kid = `test-${alg}`;
+      key.alg = alg;
+
+      await signAndVerifyWithKey(alg, key, secret);
+    });
+
+    test.each(RSA_ALGORITHMS)('verifies %s tokens', async (alg) => {
+      const { privateKey, publicKey } = await jose.generateKeyPair(alg);
+      const key = await jose.exportJWK(publicKey);
+      key.kid = `test-${alg}`;
+      key.alg = alg;
+      key.use = 'sig';
+
+      await signAndVerifyWithKey(alg, key, privateKey);
+    });
+
+    test.each(EC_ALGORITHM_CURVES)('verifies %s tokens with curve %s', async (alg, crv) => {
+      expect(EC_ALGORITHMS).toContain(alg);
+
+      const { privateKey, publicKey } = await jose.generateKeyPair(alg);
+      const key = await jose.exportJWK(publicKey);
+      key.kid = `test-${alg}`;
+      key.alg = alg;
+      key.use = 'sig';
+
+      expect(key.crv).toEqual(crv);
+      await signAndVerifyWithKey(alg, key, privateKey);
+    });
+
+    test.each(EDDSA_CURVES)('verifies EdDSA tokens with curve %s', async (crv) => {
+      const alg = 'EdDSA';
+      expect(OKP_ALGORITHMS).toContain(alg);
+
+      const { privateKey, publicKey } = await jose.generateKeyPair(alg, { crv });
+      const key = await jose.exportJWK(publicKey);
+      key.kid = `test-${alg}-${crv}`;
+      key.alg = alg;
+      key.use = 'sig';
+
+      expect(key.crv).toEqual(crv);
+      await signAndVerifyWithKey(alg, key, privateKey);
+    });
+  });
+
   test('key selection for key with kid', async () => {
     const keys = await StaticKeyCollector.importKeys([publicKeyRSA, sharedKey, sharedKey2]);
     const store = new KeyStore(keys);

package/test/src/diagnostics.test.ts

@@ -0,0 +1,151 @@
+import { DiagnosticsOptions, getSyncRulesStatus } from '@/api/diagnostics.js';
+import { RouteAPI, SlotWalBudgetInfo } from '@/api/RouteAPI.js';
+import { BucketStorageFactory } from '@/index.js';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+
+const GB = 1024 * 1024 * 1024;
+
+const MINIMAL_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id FROM test_table
+`;
+
+function makeSyncRulesContent(overrides?: { slot_name?: string }) {
+  return {
+    id: 1,
+    slot_name: overrides?.slot_name ?? 'test_slot',
+    sync_rules_content: MINIMAL_SYNC_RULES,
+    compiled_plan: null,
+    active: true,
+    storageVersion: 1,
+    last_checkpoint_lsn: 'some_lsn',
+    last_fatal_error: null,
+    last_fatal_error_ts: null,
+    last_keepalive_ts: new Date(),
+    last_checkpoint_ts: new Date(),
+    parsed(options?: any) {
+      const syncRules = SqlSyncRules.fromYaml(MINIMAL_SYNC_RULES, {
+        ...options,
+        defaultSchema: 'public'
+      });
+      return {
+        sync_rules: syncRules
+      };
+    },
+    lock() {
+      throw new Error('Not implemented in mock');
+    },
+    current_lock: undefined
+  } as any;
+}
+
+function makeBucketStorage() {
+  return {
+    getInstance() {
+      return {
+        async getStatus() {
+          return {
+            snapshot_done: true,
+            checkpoint_lsn: 'some_lsn',
+            active: true
+          };
+        }
+      };
+    }
+  } as unknown as BucketStorageFactory;
+}
+
+function makeRouteAPI(walBudget?: SlotWalBudgetInfo | undefined): RouteAPI {
+  return {
+    getParseSyncRulesOptions() {
+      return { defaultSchema: 'public' };
+    },
+    async getSourceConfig() {
+      return { tag: 'test', id: 'test', type: 'postgresql' };
+    },
+    async getConnectionStatus() {
+      return { connected: true };
+    },
+    async getDebugTablesInfo() {
+      return [];
+    },
+    async getReplicationLagBytes() {
+      return 0;
+    },
+    ...(walBudget !== undefined
+      ? {
+          async getSlotWalBudget() {
+            return walBudget;
+          }
+        }
+      : {})
+  } as unknown as RouteAPI;
+}
+
+const OPTIONS: DiagnosticsOptions = {
+  live_status: true,
+  check_connection: true,
+  include_content: false
+};
+
+describe('getSyncRulesStatus WAL budget warnings', () => {
+  test('warns when WAL budget is at 40%', async () => {
+    const api = makeRouteAPI({
+      wal_status: 'extended',
+      safe_wal_size: 4 * GB,
+      max_slot_wal_keep_size: 10 * GB
+    });
+    const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
+    const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
+    expect(walWarnings).toHaveLength(1);
+    expect(walWarnings[0].level).toBe('warning');
+    expect(walWarnings[0].message).toContain('40%');
+  });
+
+  test('no warning when WAL budget is at 80%', async () => {
+    const api = makeRouteAPI({
+      wal_status: 'reserved',
+      safe_wal_size: 8 * GB,
+      max_slot_wal_keep_size: 10 * GB
+    });
+    const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
+    const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
+    expect(walWarnings).toHaveLength(0);
+  });
+
+  test('clamps negative safe_wal_size to 0%', async () => {
+    const api = makeRouteAPI({
+      wal_status: 'unreserved',
+      safe_wal_size: -2.4 * GB,
+      max_slot_wal_keep_size: 1 * 1024 * 1024 // 1MB
+    });
+    const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
+    const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
+    expect(walWarnings).toHaveLength(1);
+    expect(walWarnings[0].message).toContain('0%');
+    expect(walWarnings[0].message).not.toMatch(/-\d+%/);
+  });
+
+  test('no WAL budget error when slot status is lost', async () => {
+    const api = makeRouteAPI({
+      wal_status: 'lost'
+    });
+    const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
+    const walErrors = result!.errors.filter(
+      (e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146')
+    );
+    expect(walErrors).toHaveLength(0);
+  });
+
+  test('no WAL error when getSlotWalBudget is not defined', async () => {
+    const api = makeRouteAPI();
+    const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
+    const walErrors = result!.errors.filter(
+      (e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146')
+    );
+    expect(walErrors).toHaveLength(0);
+  });
+});

package/test/src/module-loader.test.ts

@@ -1,5 +1,5 @@
 import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js';
-import { describe, expect, it, vi } from 'vitest';
+import { describe, expect, it } from 'vitest';
 
 interface MockConfig {
   connections?: { type: string }[];

package/test/src/routes/mocks.ts

@@ -1,3 +1,4 @@
+import { EventsEngine } from '@/events/EventsEngine.js';
 import {
   BucketStorageFactory,
   createCoreAPIMetrics,
@@ -11,7 +12,6 @@ import {
   SyncRulesBucketStorage
 } from '@/index.js';
 import { MeterProvider } from '@opentelemetry/sdk-metrics';
-import { EventsEngine } from '@/events/EventsEngine.js';
 
 export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
   // This is very incomplete - just enough to get the current tests passing.

package/test/src/routes/stream.test.ts

@@ -1,6 +1,6 @@
 import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
 import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { DEFAULT_HYDRATION_STATE, SqlSyncRules } from '@powersync/service-sync-rules';
 import { Readable, Writable } from 'stream';
 import { pipeline } from 'stream/promises';
 import { describe, expect, it } from 'vitest';
@@ -8,7 +8,6 @@ import winston from 'winston';
 import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
 import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
 import { mockServiceContext } from './mocks.js';
-import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
 
 describe('Stream Route', () => {
   describe('compressed stream', () => {