@powersync/service-core 1.18.2 → 1.19.1

package/src/sync/BucketChecksumState.ts

@@ -2,38 +2,32 @@ import {
   BucketDescription,
   BucketPriority,
   BucketSource,
+  HydratedSyncRules,
   RequestedStream,
-  RequestJwtPayload,
   RequestParameters,
-  ResolvedBucket,
-  SqlSyncRules
+  ResolvedBucket
 } from '@powersync/service-sync-rules';
 
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import {
+  logger as defaultLogger,
   ErrorCode,
   Logger,
   ServiceAssertionError,
-  ServiceError,
-  logger as defaultLogger
+  ServiceError
 } from '@powersync/lib-services-framework';
-import { JSONBig } from '@powersync/service-jsonbig';
 import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
+import { JwtPayload } from '../auth/JwtPayload.js';
 import { SyncContext } from './SyncContext.js';
 import { getIntersection, hasIntersection } from './util.js';
 
-export interface VersionedSyncRules {
-  syncRules: SqlSyncRules;
-  version: number;
-}
-
 export interface BucketChecksumStateOptions {
   syncContext: SyncContext;
   bucketStorage: BucketChecksumStateStorage;
-  syncRules: VersionedSyncRules;
-  tokenPayload: RequestJwtPayload;
+  syncRules: HydratedSyncRules;
+  tokenPayload: JwtPayload;
   syncRequest: util.StreamingSyncRequest;
   logger?: Logger;
 }
@@ -118,7 +112,7 @@ export class BucketChecksumState {
    */
   async buildNextCheckpointLine(next: storage.StorageCheckpointUpdate): Promise<CheckpointLine | null> {
     const { writeCheckpoint, base } = next;
-    const user_id = this.parameterState.syncParams.userId;
+    const userIdForLogs = this.parameterState.syncParams.userId;
 
     const storage = this.bucketStorage;
 
@@ -226,7 +220,7 @@ export class BucketChecksumState {
         message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
         this.logger.info(message, {
           checkpoint: base.checkpoint,
-          user_id: user_id,
+          user_id: userIdForLogs,
           buckets: allBuckets.length,
           updated: diff.updatedBuckets.length,
           removed: diff.removedBuckets.length
@@ -245,7 +239,7 @@ export class BucketChecksumState {
       deferredLog = () => {
         let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
         message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
-        this.logger.info(message, { checkpoint: base.checkpoint, user_id: user_id, buckets: allBuckets.length });
+        this.logger.info(message, { checkpoint: base.checkpoint, user_id: userIdForLogs, buckets: allBuckets.length });
       };
       bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
 
@@ -253,7 +247,7 @@ export class BucketChecksumState {
       const streamNameToIndex = new Map<string, number>();
       this.streamNameToIndex = streamNameToIndex;
 
-      for (const source of this.parameterState.syncRules.syncRules.bucketSources) {
+      for (const source of this.parameterState.syncRules.definition.bucketSources) {
         if (this.parameterState.isSubscribedToStream(source)) {
           streamNameToIndex.set(source.name, subscriptions.length);
 
@@ -381,7 +375,7 @@ export interface CheckpointUpdate {
 export class BucketParameterState {
   private readonly context: SyncContext;
   public readonly bucketStorage: BucketChecksumStateStorage;
-  public readonly syncRules: VersionedSyncRules;
+  public readonly syncRules: HydratedSyncRules;
   public readonly syncParams: RequestParameters;
   private readonly querier: BucketParameterQuerier;
   /**
@@ -399,13 +393,13 @@ export class BucketParameterState {
   private cachedDynamicBuckets: ResolvedBucket[] | null = null;
   private cachedDynamicBucketSet: Set<string> | null = null;
 
-  private readonly lookups: Set<string>;
+  private lookupsFromPreviousCheckpoint: Set<string> | null = null;
 
   constructor(
     context: SyncContext,
     bucketStorage: BucketChecksumStateStorage,
-    syncRules: VersionedSyncRules,
-    tokenPayload: RequestJwtPayload,
+    syncRules: HydratedSyncRules,
+    tokenPayload: JwtPayload,
     request: util.StreamingSyncRequest,
     logger: Logger
   ) {
@@ -436,11 +430,10 @@ export class BucketParameterState {
     this.includeDefaultStreams = subscriptions?.include_defaults ?? true;
     this.explicitStreamSubscriptions = explicitStreamSubscriptions;
 
-    const { querier, errors } = syncRules.syncRules.getBucketParameterQuerier({
+    const { querier, errors } = syncRules.getBucketParameterQuerier({
       globalParameters: this.syncParams,
       hasDefaultStreams: this.includeDefaultStreams,
-      streams: streamsByName,
-      bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${syncRules.version}`)
+      streams: streamsByName
     });
     this.querier = querier;
     this.streamErrors = Object.groupBy(errors, (e) => e.descriptor) as Record<string, QuerierError[]>;
@@ -448,7 +441,6 @@ export class BucketParameterState {
     this.staticBuckets = new Map<string, ResolvedBucket>(
       mergeBuckets(this.querier.staticBuckets).map((b) => [b.bucket, b])
     );
-    this.lookups = new Set<string>(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values)));
     this.subscribedStreamNames = new Set(Object.keys(streamsByName));
   }
 
@@ -547,7 +539,6 @@ export class BucketParameterState {
    */
   private async getCheckpointUpdateDynamic(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
     const querier = this.querier;
-    const storage = this.bucketStorage;
     const staticBuckets = this.staticBuckets.values();
     const update = checkpoint.update;
 
@@ -561,10 +552,10 @@ export class BucketParameterState {
       invalidateDataBuckets = true;
     }
 
-    if (update.invalidateParameterBuckets) {
+    if (update.invalidateParameterBuckets || this.lookupsFromPreviousCheckpoint == null) {
       hasParameterChange = true;
     } else {
-      if (hasIntersection(this.lookups, update.updatedParameterLookups)) {
+      if (hasIntersection(this.lookupsFromPreviousCheckpoint, update.updatedParameterLookups)) {
         // This is a very coarse re-check of all queries
         hasParameterChange = true;
       }
@@ -572,13 +563,20 @@ export class BucketParameterState {
 
     let dynamicBuckets: ResolvedBucket[];
     if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
+      const recordedLookups = new Set<string>();
+
       dynamicBuckets = await querier.queryDynamicBucketDescriptions({
         getParameterSets(lookups) {
+          for (const lookup of lookups) {
+            recordedLookups.add(lookup.serializedRepresentation);
+          }
+
           return checkpoint.base.getParameterSets(lookups);
         }
       });
       this.cachedDynamicBuckets = dynamicBuckets;
       this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
+      this.lookupsFromPreviousCheckpoint = recordedLookups;
       invalidateDataBuckets = true;
     } else {
       dynamicBuckets = this.cachedDynamicBuckets;

package/src/sync/sync.ts

@@ -1,5 +1,5 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import { BucketDescription, BucketPriority, RequestJwtPayload } from '@powersync/service-sync-rules';
+import { BucketDescription, BucketPriority, HydratedSyncRules, SqliteJsonValue } from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -9,7 +9,7 @@ import * as util from '../util/util-index.js';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
-import { BucketChecksumState, CheckpointLine, VersionedSyncRules } from './BucketChecksumState.js';
+import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
 import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
 import { SyncContext } from './SyncContext.js';
 import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
@@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr
 export interface SyncStreamParameters {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
-  syncRules: VersionedSyncRules;
+  syncRules: HydratedSyncRules;
   params: util.StreamingSyncRequest;
   token: auth.JwtPayload;
   logger?: Logger;
@@ -94,18 +94,15 @@ export async function* streamResponse(
 async function* streamResponseInner(
   syncContext: SyncContext,
   bucketStorage: storage.SyncRulesBucketStorage,
-  syncRules: VersionedSyncRules,
+  syncRules: HydratedSyncRules,
   params: util.StreamingSyncRequest,
-  tokenPayload: RequestJwtPayload,
+  tokenPayload: auth.JwtPayload,
   tracker: RequestTracker,
   signal: AbortSignal,
   logger: Logger,
   isEncodingAsBson: boolean
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
-  const { raw_data } = params;
-
-  const userId = tokenPayload.sub;
-  const checkpointUserId = util.checkpointUserId(userId as string, params.client_id);
+  const checkpointUserId = util.checkpointUserId(tokenPayload.userIdString, params.client_id);
 
   const checksumState = new BucketChecksumState({
     syncContext,
@@ -236,7 +233,7 @@ async function* streamResponseInner(
           onRowsSent: markOperationsSent,
           abort_connection: signal,
           abort_batch: abortCheckpointSignal,
-          user_id: userId,
+          userIdForLogs: tokenPayload.userIdJson,
           // Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial
           // sync complete message instead.
           forPriority: !isLast ? priority : null,
@@ -270,7 +267,8 @@ interface BucketDataRequest {
    * This signal also fires when abort_connection fires.
    */
   abort_batch: AbortSignal;
-  user_id?: string;
+  /** User id for debug purposes, not for sync rules. */
+  userIdForLogs?: SqliteJsonValue;
   forPriority: BucketPriority | null;
   onRowsSent: (stats: OperationsSentStats) => void;
   logger: Logger;
@@ -333,7 +331,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
   let checkpointInvalidated = false;
 
   if (syncContext.syncSemaphore.isLocked()) {
-    logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
+    logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.userIdForLogs });
   }
   const acquired = await acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
   if (acquired === 'aborted') {
@@ -346,7 +344,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       // This can be noisy, so we only log when we get close to the
       // concurrency limit.
       logger.info(`Got sync lock. Slots available: ${value - 1}`, {
-        user_id: request.user_id,
+        user_id: request.userIdForLogs,
         sync_data_slots: value - 1
       });
     }
@@ -429,7 +427,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       // This can be noisy, so we only log when we get close to the
       // concurrency limit.
       logger.info(`Releasing sync lock`, {
-        user_id: request.user_id
+        user_id: request.userIdForLogs
       });
     }
     release();

package/src/util/config/collectors/config-collector.ts

@@ -70,13 +70,20 @@ export abstract class ConfigCollector {
         return this.parseJSON(content);
       default: {
         // No content type provided, need to try both
+        let yamlError: unknown;
         try {
           return this.parseYaml(content);
-        } catch (ex) {}
+        } catch (ex) {
+          yamlError = ex;
+        }
         try {
           return this.parseJSON(content);
         } catch (ex) {
-          throw new Error(`Could not parse PowerSync config file content as JSON or YAML: ${ex}`);
+          throw new Error(
+            `Could not parse PowerSync config file content as JSON or YAML: JSON Error: ${ex}${
+              yamlError ? `\nYAML Error: ${yamlError}` : ''
+            }`
+          );
         }
       }
     }

package/src/util/config.ts

@@ -1,15 +1,33 @@
 import * as fs from 'fs/promises';
+import winston from 'winston';
 
-import { container } from '@powersync/lib-services-framework';
+import { container, logger, LogFormat, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT } from '@powersync/lib-services-framework';
+import { configFile } from '@powersync/service-types';
 import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
 import { CompoundConfigCollector } from './util-index.js';
 
+export function configureLogger(config?: configFile.LoggingConfig): void {
+  const level = process.env.PS_LOG_LEVEL ?? config?.level ?? DEFAULT_LOG_LEVEL;
+  const format =
+    (process.env.PS_LOG_FORMAT as configFile.LoggingConfig['format']) ?? config?.format ?? DEFAULT_LOG_FORMAT;
+  const winstonFormat = format === 'json' ? LogFormat.production : LogFormat.development;
+
+  // We want the user to always be aware that a log level was configured (they might forget they set it in the config and wonder why they aren't seeing logs)
+  // We log this using the configured format, but before we configure the level.
+  logger.configure({ level: DEFAULT_LOG_LEVEL, format: winstonFormat, transports: [new winston.transports.Console()] });
+  logger.info(`Configured logger with level "${level}" and format "${format}"`);
+
+  logger.configure({ level, format: winstonFormat, transports: [new winston.transports.Console()] });
+}
+
 /**
  * Loads the resolved config using the registered config collector
  */
 export async function loadConfig(runnerConfig: RunnerConfig) {
   const collector = container.getImplementation(CompoundConfigCollector);
-  return collector.collectConfig(runnerConfig);
+  const config = await collector.collectConfig(runnerConfig);
+  configureLogger(config.base_config.system?.logging);
+  return config;
 }
 
 export async function loadSyncRules(config: ResolvedPowerSyncConfig): Promise<string | undefined> {

package/test/src/auth.test.ts

@@ -70,7 +70,7 @@ describe('JWT Auth', () => {
       defaultAudiences: ['tests'],
       maxAge: '6m'
     });
-    expect(verified.sub).toEqual('f1');
+    expect(verified.userIdJson).toEqual('f1');
     await expect(
       store.verifyJwt(signedJwt, {
         defaultAudiences: ['other'],
@@ -126,6 +126,58 @@ describe('JWT Auth', () => {
     ).rejects.toThrow('[PSYNC_S2103] JWT has expired');
   });
 
+  test('KeyStore normalizes sub claim values', async () => {
+    const keys = await StaticKeyCollector.importKeys([sharedKey]);
+    const store = new KeyStore(keys);
+    const signKey = (await jose.importJWK(sharedKey)) as jose.KeyLike;
+
+    const signWithSub = async (sub: any) => {
+      return new jose.SignJWT({ sub })
+        .setProtectedHeader({ alg: 'HS256', kid: 'k1' })
+        .setIssuedAt()
+        .setIssuer('tester')
+        .setAudience('tests')
+        .setExpirationTime('5m')
+        .sign(signKey);
+    };
+
+    const stringToken = await signWithSub('user-1');
+    const stringVerified = await store.verifyJwt(stringToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    expect(stringVerified.userIdJson).toEqual('user-1');
+    expect(stringVerified.userIdString).toEqual('user-1');
+
+    const booleanToken = await signWithSub(true);
+    const booleanVerified = await store.verifyJwt(booleanToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    expect(booleanVerified.userIdJson).toEqual(1n);
+    expect(booleanVerified.userIdString).toEqual('1');
+
+    const objectSub = { id: 1, name: 'test' };
+    const objectToken = await signWithSub(objectSub);
+    const objectVerified = await store.verifyJwt(objectToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    // The exact JSON serialization here may change in the future
+    expect(objectVerified.userIdJson).toEqual(`{"id":1.0,"name":"test"}`);
+    expect(objectVerified.userIdString).toEqual(`{"id":1.0,"name":"test"}`);
+
+    const arraySub = [1, 'a'];
+    const arrayToken = await signWithSub(arraySub);
+    const arrayVerified = await store.verifyJwt(arrayToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    // The exact JSON serialization here may change in the future
+    expect(arrayVerified.userIdJson).toEqual(`[1.0,"a"]`);
+    expect(arrayVerified.userIdString).toEqual(`[1.0,"a"]`);
+  });
+
   test('Algorithm validation', async () => {
     const keys = await StaticKeyCollector.importKeys([publicKeyRSA]);
     const store = new KeyStore(keys);
@@ -210,12 +262,14 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey2);
 
-    await expect(
-      store.verifyJwt(signedJwt3, {
-        defaultAudiences: ['tests'],
-        maxAge: '6m'
-      })
-    ).resolves.toMatchObject({ sub: 'f1' });
+    expect(
+      (
+        await store.verifyJwt(signedJwt3, {
+          defaultAudiences: ['tests'],
+          maxAge: '6m'
+        })
+      ).parsedPayload
+    ).toMatchObject({ sub: 'f1' });
 
     // Random kid, matches sharedKey2
     const signedJwt4 = await new jose.SignJWT({})
@@ -227,12 +281,14 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey2);
 
-    await expect(
-      store.verifyJwt(signedJwt4, {
-        defaultAudiences: ['tests'],
-        maxAge: '6m'
-      })
-    ).resolves.toMatchObject({ sub: 'f1' });
+    expect(
+      (
+        await store.verifyJwt(signedJwt4, {
+          defaultAudiences: ['tests'],
+          maxAge: '6m'
+        })
+      ).parsedPayload
+    ).toMatchObject({ sub: 'f1' });
   });
 
   test('KeyOptions', async () => {
@@ -258,7 +314,7 @@ describe('JWT Auth', () => {
       defaultAudiences: ['tests'],
       maxAge: '6m'
     });
-    expect(verified.sub).toEqual('f1');
+    expect(verified.userIdJson).toEqual('f1');
 
     const signedJwt2 = await new jose.SignJWT({})
       .setProtectedHeader({ alg: 'HS256', kid: 'k1' })
@@ -410,12 +466,12 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey);
 
-    const verified = (await store.verifyJwt(signedJwt, {
+    const verified = await store.verifyJwt(signedJwt, {
       defaultAudiences: ['tests'],
       maxAge: '6m'
-    })) as JwtPayload & { claim: string };
+    });
 
-    expect(verified.claim).toEqual('test-claim');
+    expect(verified.parsedPayload.claim).toEqual('test-claim');
   });
 
   test('signing with ECDSA', async () => {
@@ -432,12 +488,12 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey);
 
-    const verified = (await store.verifyJwt(signedJwt, {
+    const verified = await store.verifyJwt(signedJwt, {
       defaultAudiences: ['tests'],
       maxAge: '6m'
-    })) as JwtPayload & { claim: string };
+    });
 
-    expect(verified.claim).toEqual('test-claim-2');
+    expect(verified.parsedPayload.claim).toEqual('test-claim-2');
   });
 
   describe('debugKeyNotFound', () => {

package/test/src/config.test.ts

@@ -69,4 +69,21 @@ describe('Config', () => {
 
     expect(config.api_parameters.max_buckets_per_connection).toBe(1);
   });
+  it('should throw YAML validation error for invalid base64 config', {}, async () => {
+    const yamlConfig = /* yaml */ `
+      # PowerSync config
+      replication:
+        connections: []
+      storage:
+        type: !env INVALID_VAR
+    `;
+
+    const collector = new CompoundConfigCollector();
+
+    await expect(
+      collector.collectConfig({
+        config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
+      })
+    ).rejects.toThrow(/YAML Error:[\s\S]*Attempting to substitute environment variable INVALID_VAR/);
+  });
 });

package/test/src/routes/stream.test.ts

@@ -1,4 +1,4 @@
-import { BasicRouterRequest, Context, SyncRulesBucketStorage } from '@/index.js';
+import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
 import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { Readable, Writable } from 'stream';
@@ -15,11 +15,11 @@ describe('Stream Route', () => {
       const context: Context = {
         logger: logger,
         service_context: mockServiceContext(null),
-        token_payload: {
+        token_payload: new JwtPayload({
           sub: '',
           exp: 0,
           iat: 0
-        }
+        })
       };
 
       const request: BasicRouterRequest = {
@@ -45,7 +45,7 @@ describe('Stream Route', () => {
 
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}');
+          return new SqlSyncRules('bucket_definitions: {}').hydrate();
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');
@@ -56,11 +56,11 @@ describe('Stream Route', () => {
       const context: Context = {
         logger: logger,
         service_context: serviceContext,
-        token_payload: {
+        token_payload: new JwtPayload({
           exp: new Date().getTime() / 1000 + 10000,
           iat: new Date().getTime() / 1000 - 10000,
           sub: 'test-user'
-        }
+        })
       };
 
       // It may be worth eventually doing this via Fastify to test the full stack
@@ -83,7 +83,7 @@ describe('Stream Route', () => {
     it('logs the application metadata', async () => {
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}');
+          return new SqlSyncRules('bucket_definitions: {}').hydrate();
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');
@@ -108,11 +108,11 @@ describe('Stream Route', () => {
       const context: Context = {
         logger: testLogger,
         service_context: serviceContext,
-        token_payload: {
+        token_payload: new JwtPayload({
           exp: new Date().getTime() / 1000 + 10000,
           iat: new Date().getTime() / 1000 - 10000,
           sub: 'test-user'
-        }
+        })
       };
 
       const request: BasicRouterRequest = {