@powersync/service-core 0.0.0-dev-20260203155513 → 0.0.0-dev-20260223080959

package/src/sync/BucketChecksumState.ts

@@ -1,10 +1,11 @@
 import {
   BucketDescription,
+  BucketParameterQuerier,
   BucketPriority,
   BucketSource,
   HydratedSyncRules,
+  QuerierError,
   RequestedStream,
-  RequestJwtPayload,
   RequestParameters,
   ResolvedBucket
 } from '@powersync/service-sync-rules';
@@ -13,13 +14,13 @@ import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import {
-  ErrorCode,
   logger as defaultLogger,
+  ErrorCode,
   Logger,
   ServiceAssertionError,
   ServiceError
 } from '@powersync/lib-services-framework';
-import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
+import { JwtPayload } from '../auth/JwtPayload.js';
 import { SyncContext } from './SyncContext.js';
 import { getIntersection, hasIntersection } from './util.js';
 
@@ -27,7 +28,7 @@ export interface BucketChecksumStateOptions {
   syncContext: SyncContext;
   bucketStorage: BucketChecksumStateStorage;
   syncRules: HydratedSyncRules;
-  tokenPayload: RequestJwtPayload;
+  tokenPayload: JwtPayload;
   syncRequest: util.StreamingSyncRequest;
   logger?: Logger;
 }
@@ -79,7 +80,6 @@ export class BucketChecksumState {
   private pendingBucketDownloads = new Set<string>();
 
   private readonly logger: Logger;
-  private resync = false;
 
   constructor(options: BucketChecksumStateOptions) {
     this.context = options.syncContext;
@@ -113,13 +113,12 @@ export class BucketChecksumState {
    */
   async buildNextCheckpointLine(next: storage.StorageCheckpointUpdate): Promise<CheckpointLine | null> {
     const { writeCheckpoint, base } = next;
-    const user_id = this.parameterState.syncParams.userId;
+    const userIdForLogs = this.parameterState.syncParams.userId;
 
     const storage = this.bucketStorage;
 
     const update = await this.parameterState.getCheckpointUpdate(next);
     const { buckets: allBuckets, updatedBuckets } = update;
-    this.resync = updatedBuckets === INVALIDATE_ALL_BUCKETS;
 
     /** Set of all buckets in this checkpoint. */
     const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
@@ -164,7 +163,6 @@ export class BucketChecksumState {
       checksumMap = newChecksums;
     } else {
       // Re-check all buckets
-      this.resync = true;
       const bucketList = [...bucketDescriptionMap.keys()];
       checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
     }
@@ -189,7 +187,6 @@ export class BucketChecksumState {
         diff.updatedBuckets.length == 0
       ) {
         // No changes - don't send anything to the client
-        this.resync = false;
         return null;
       }
 
@@ -224,7 +221,7 @@ export class BucketChecksumState {
         message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
         this.logger.info(message, {
           checkpoint: base.checkpoint,
-          user_id: user_id,
+          user_id: userIdForLogs,
           buckets: allBuckets.length,
           updated: diff.updatedBuckets.length,
           removed: diff.removedBuckets.length
@@ -243,7 +240,7 @@ export class BucketChecksumState {
       deferredLog = () => {
         let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
         message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
-        this.logger.info(message, { checkpoint: base.checkpoint, user_id: user_id, buckets: allBuckets.length });
+        this.logger.info(message, { checkpoint: base.checkpoint, user_id: userIdForLogs, buckets: allBuckets.length });
       };
       bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
 
@@ -398,12 +395,12 @@ export class BucketParameterState {
   private cachedDynamicBucketSet: Set<string> | null = null;
 
   private lookupsFromPreviousCheckpoint: Set<string> | null = null;
-  private resyncBucket = false;
+
   constructor(
     context: SyncContext,
     bucketStorage: BucketChecksumStateStorage,
     syncRules: HydratedSyncRules,
-    tokenPayload: RequestJwtPayload,
+    tokenPayload: JwtPayload,
     request: util.StreamingSyncRequest,
     logger: Logger
   ) {

package/src/sync/sync.ts

@@ -1,31 +1,19 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import { BucketDescription, BucketPriority, HydratedSyncRules, RequestJwtPayload } from '@powersync/service-sync-rules';
+import { BucketDescription, BucketPriority, HydratedSyncRules, SqliteJsonValue } from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 
 import * as auth from '../auth/auth-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
+
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
 import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
 import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
 import { SyncContext } from './SyncContext.js';
-import {
-  acquireSemaphoreAbortable,
-  parseCheckpointLineForEvent,
-  settledPromise,
-  tokenStream,
-  TokenStreamOptions
-} from './util.js';
-import { EventsEngine } from '../events/EventsEngine.js';
-import { event_types } from '@powersync/service-types';
-
-type Event = {
-  engine?: EventsEngine;
-  user_id: string;
-  client_id: string;
-};
+import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
+
 export interface SyncStreamParameters {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
@@ -41,7 +29,6 @@ export interface SyncStreamParameters {
   tokenStreamOptions?: Partial<TokenStreamOptions>;
 
   tracker: RequestTracker;
-  event?: Event;
 }
 
 export async function* streamResponse(
@@ -56,8 +43,7 @@ export async function* streamResponse(
     tokenStreamOptions,
     tracker,
     signal,
-    isEncodingAsBson,
-    event
+    isEncodingAsBson
   } = options;
   const logger = options.logger ?? defaultLogger;
 
@@ -85,8 +71,7 @@ export async function* streamResponse(
     tracker,
     controller.signal,
     logger,
-    isEncodingAsBson,
-    event
+    isEncodingAsBson
   );
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
@@ -111,17 +96,13 @@ async function* streamResponseInner(
   bucketStorage: storage.SyncRulesBucketStorage,
   syncRules: HydratedSyncRules,
   params: util.StreamingSyncRequest,
-  tokenPayload: RequestJwtPayload,
+  tokenPayload: auth.JwtPayload,
   tracker: RequestTracker,
   signal: AbortSignal,
   logger: Logger,
-  isEncodingAsBson: boolean,
-  event?: Event
+  isEncodingAsBson: boolean
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
-  const { raw_data } = params;
-
-  const userId = tokenPayload.sub;
-  const checkpointUserId = util.checkpointUserId(userId as string, params.client_id);
+  const checkpointUserId = util.checkpointUserId(tokenPayload.userIdString, params.client_id);
 
   const checksumState = new BucketChecksumState({
     syncContext,
@@ -179,15 +160,6 @@ async function* streamResponseInner(
 
       // Since yielding can block, we update the state just before yielding the line.
       line.advance();
-      parseCheckpointLineForEvent(checkpointLine);
-
-      event?.engine?.emit(event_types.EventsEngineEventType.SYNC_ANALYTICS_EVENT, {
-        user_id: event?.user_id,
-        client_id: event?.client_id,
-        request_streams: params.streams,
-        data: parseCheckpointLineForEvent(checkpointLine)
-      });
-
       yield checkpointLine;
 
       // Start syncing data for buckets up to the checkpoint. As soon as we have completed at least one priority and
@@ -261,7 +233,7 @@ async function* streamResponseInner(
           onRowsSent: markOperationsSent,
           abort_connection: signal,
           abort_batch: abortCheckpointSignal,
-          user_id: userId,
+          userIdForLogs: tokenPayload.userIdJson,
           // Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial
           // sync complete message instead.
           forPriority: !isLast ? priority : null,
@@ -295,7 +267,8 @@ interface BucketDataRequest {
    * This signal also fires when abort_connection fires.
    */
   abort_batch: AbortSignal;
-  user_id?: string;
+  /** User id for debug purposes, not for sync rules. */
+  userIdForLogs?: SqliteJsonValue;
   forPriority: BucketPriority | null;
   onRowsSent: (stats: OperationsSentStats) => void;
   logger: Logger;
@@ -358,7 +331,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
   let checkpointInvalidated = false;
 
   if (syncContext.syncSemaphore.isLocked()) {
-    logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
+    logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.userIdForLogs });
   }
   const acquired = await acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
   if (acquired === 'aborted') {
@@ -371,7 +344,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       // This can be noisy, so we only log when we get close to the
       // concurrency limit.
       logger.info(`Got sync lock. Slots available: ${value - 1}`, {
-        user_id: request.user_id,
+        user_id: request.userIdForLogs,
         sync_data_slots: value - 1
       });
     }
@@ -454,7 +427,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       // This can be noisy, so we only log when we get close to the
       // concurrency limit.
       logger.info(`Releasing sync lock`, {
-        user_id: request.user_id
+        user_id: request.userIdForLogs
       });
     }
     release();

package/src/sync/util.ts

@@ -2,11 +2,8 @@ import * as timers from 'timers/promises';
 
 import { SemaphoreInterface } from 'async-mutex';
 import * as util from '../util/util-index.js';
-import { StreamingSyncCheckpoint, StreamingSyncCheckpointDiff } from '../util/util-index.js';
 import { RequestTracker } from './RequestTracker.js';
-import * as bson from 'bson';
 import { serialize } from 'bson';
-import { SyncEventCheckpointType } from '@powersync/service-types/dist/reports.js';
 
 export type TokenStreamOptions = {
   /**
@@ -218,25 +215,3 @@ export function* getIntersection<T>(a: MapOrSet<T>, b: MapOrSet<T>): IterableIte
     }
   }
 }
-
-export function parseCheckpointLineForEvent(line: StreamingSyncCheckpoint | StreamingSyncCheckpointDiff) {
-  if ('checkpoint' in line) {
-    return {
-      id: new bson.ObjectId(),
-      type: SyncEventCheckpointType.FULL,
-      last_op_id: line.checkpoint.last_op_id,
-      buckets: line.checkpoint.buckets,
-      streams: line.checkpoint.streams,
-      date: new Date()
-    };
-  } else {
-    return {
-      id: new bson.ObjectId(),
-      type: SyncEventCheckpointType.DIFF,
-      last_op_id: line.checkpoint_diff.last_op_id,
-      updated_buckets: line.checkpoint_diff.updated_buckets,
-      removed_buckets: line.checkpoint_diff.removed_buckets,
-      date: new Date()
-    };
-  }
-}

package/src/util/config.ts

@@ -8,11 +8,16 @@ import { CompoundConfigCollector } from './util-index.js';
 
 export function configureLogger(config?: configFile.LoggingConfig): void {
   const level = process.env.PS_LOG_LEVEL ?? config?.level ?? DEFAULT_LOG_LEVEL;
-  const format = (process.env.PS_LOG_FORMAT as configFile.LoggingConfig['format']) ?? config?.format ?? DEFAULT_LOG_FORMAT;
+  const format =
+    (process.env.PS_LOG_FORMAT as configFile.LoggingConfig['format']) ?? config?.format ?? DEFAULT_LOG_FORMAT;
   const winstonFormat = format === 'json' ? LogFormat.production : LogFormat.development;
 
+  // We want the user to always be aware that a log level was configured (they might forget they set it in the config and wonder why they aren't seeing logs)
+  // We log this using the configured format, but before we configure the level.
+  logger.configure({ level: DEFAULT_LOG_LEVEL, format: winstonFormat, transports: [new winston.transports.Console()] });
+  logger.info(`Configured logger with level "${level}" and format "${format}"`);
+
   logger.configure({ level, format: winstonFormat, transports: [new winston.transports.Console()] });
-  console.log(`[PREFLIGHT] Configured logger with level "${level}" and format "${format}"`); // we want the user to always be aware that a log level was configured (they might forget they set it in the config and wonder why they aren't seeing logs)
 }
 
 /**

package/test/src/auth.test.ts

@@ -70,7 +70,7 @@ describe('JWT Auth', () => {
       defaultAudiences: ['tests'],
       maxAge: '6m'
     });
-    expect(verified.sub).toEqual('f1');
+    expect(verified.userIdJson).toEqual('f1');
     await expect(
       store.verifyJwt(signedJwt, {
         defaultAudiences: ['other'],
@@ -126,6 +126,58 @@ describe('JWT Auth', () => {
     ).rejects.toThrow('[PSYNC_S2103] JWT has expired');
   });
 
+  test('KeyStore normalizes sub claim values', async () => {
+    const keys = await StaticKeyCollector.importKeys([sharedKey]);
+    const store = new KeyStore(keys);
+    const signKey = (await jose.importJWK(sharedKey)) as jose.KeyLike;
+
+    const signWithSub = async (sub: any) => {
+      return new jose.SignJWT({ sub })
+        .setProtectedHeader({ alg: 'HS256', kid: 'k1' })
+        .setIssuedAt()
+        .setIssuer('tester')
+        .setAudience('tests')
+        .setExpirationTime('5m')
+        .sign(signKey);
+    };
+
+    const stringToken = await signWithSub('user-1');
+    const stringVerified = await store.verifyJwt(stringToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    expect(stringVerified.userIdJson).toEqual('user-1');
+    expect(stringVerified.userIdString).toEqual('user-1');
+
+    const booleanToken = await signWithSub(true);
+    const booleanVerified = await store.verifyJwt(booleanToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    expect(booleanVerified.userIdJson).toEqual(1n);
+    expect(booleanVerified.userIdString).toEqual('1');
+
+    const objectSub = { id: 1, name: 'test' };
+    const objectToken = await signWithSub(objectSub);
+    const objectVerified = await store.verifyJwt(objectToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    // The exact JSON serialization here may change in the future
+    expect(objectVerified.userIdJson).toEqual(`{"id":1.0,"name":"test"}`);
+    expect(objectVerified.userIdString).toEqual(`{"id":1.0,"name":"test"}`);
+
+    const arraySub = [1, 'a'];
+    const arrayToken = await signWithSub(arraySub);
+    const arrayVerified = await store.verifyJwt(arrayToken, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    });
+    // The exact JSON serialization here may change in the future
+    expect(arrayVerified.userIdJson).toEqual(`[1.0,"a"]`);
+    expect(arrayVerified.userIdString).toEqual(`[1.0,"a"]`);
+  });
+
   test('Algorithm validation', async () => {
     const keys = await StaticKeyCollector.importKeys([publicKeyRSA]);
     const store = new KeyStore(keys);
@@ -210,12 +262,14 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey2);
 
-    await expect(
-      store.verifyJwt(signedJwt3, {
-        defaultAudiences: ['tests'],
-        maxAge: '6m'
-      })
-    ).resolves.toMatchObject({ sub: 'f1' });
+    expect(
+      (
+        await store.verifyJwt(signedJwt3, {
+          defaultAudiences: ['tests'],
+          maxAge: '6m'
+        })
+      ).parsedPayload
+    ).toMatchObject({ sub: 'f1' });
 
     // Random kid, matches sharedKey2
     const signedJwt4 = await new jose.SignJWT({})
@@ -227,12 +281,14 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey2);
 
-    await expect(
-      store.verifyJwt(signedJwt4, {
-        defaultAudiences: ['tests'],
-        maxAge: '6m'
-      })
-    ).resolves.toMatchObject({ sub: 'f1' });
+    expect(
+      (
+        await store.verifyJwt(signedJwt4, {
+          defaultAudiences: ['tests'],
+          maxAge: '6m'
+        })
+      ).parsedPayload
+    ).toMatchObject({ sub: 'f1' });
   });
 
   test('KeyOptions', async () => {
@@ -258,7 +314,7 @@ describe('JWT Auth', () => {
       defaultAudiences: ['tests'],
       maxAge: '6m'
     });
-    expect(verified.sub).toEqual('f1');
+    expect(verified.userIdJson).toEqual('f1');
 
     const signedJwt2 = await new jose.SignJWT({})
       .setProtectedHeader({ alg: 'HS256', kid: 'k1' })
@@ -410,12 +466,12 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey);
 
-    const verified = (await store.verifyJwt(signedJwt, {
+    const verified = await store.verifyJwt(signedJwt, {
       defaultAudiences: ['tests'],
       maxAge: '6m'
-    })) as JwtPayload & { claim: string };
+    });
 
-    expect(verified.claim).toEqual('test-claim');
+    expect(verified.parsedPayload.claim).toEqual('test-claim');
   });
 
   test('signing with ECDSA', async () => {
@@ -432,12 +488,12 @@ describe('JWT Auth', () => {
       .setExpirationTime('5m')
       .sign(signKey);
 
-    const verified = (await store.verifyJwt(signedJwt, {
+    const verified = await store.verifyJwt(signedJwt, {
       defaultAudiences: ['tests'],
       maxAge: '6m'
-    })) as JwtPayload & { claim: string };
+    });
 
-    expect(verified.claim).toEqual('test-claim-2');
+    expect(verified.parsedPayload.claim).toEqual('test-claim-2');
   });
 
   describe('debugKeyNotFound', () => {

package/test/src/routes/admin.test.ts

@@ -0,0 +1,48 @@
+import { BasicRouterRequest, Context, JwtPayload } from '@/index.js';
+import { logger } from '@powersync/lib-services-framework';
+import { describe, expect, it } from 'vitest';
+import { validate } from '../../../src/routes/endpoints/admin.js';
+import { mockServiceContext } from './mocks.js';
+
+describe('admin routes', () => {
+  describe('validate', () => {
+    it('reports errors with source location', async () => {
+      const context: Context = {
+        logger: logger,
+        service_context: mockServiceContext(null),
+        token_payload: new JwtPayload({
+          sub: '',
+          exp: 0,
+          iat: 0
+        })
+      };
+
+      const request: BasicRouterRequest = {
+        headers: {},
+        hostname: '',
+        protocol: 'http'
+      };
+
+      const response = await validate.handler({
+        context,
+        params: {
+          sync_rules: `
+bucket_definitions:
+  missing_table:
+    data:
+      - SELECT * FROM missing_table
+`
+        },
+        request
+      });
+
+      expect(response.errors).toEqual([
+        expect.objectContaining({
+          level: 'warning',
+          location: { start_offset: 70, end_offset: 83 },
+          message: 'Table public.missing_table not found'
+        })
+      ]);
+    });
+  });
+});

package/test/src/routes/mocks.ts

@@ -41,8 +41,29 @@ export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | nu
         return {
           getParseSyncRulesOptions() {
             return { defaultSchema: 'public' };
+          },
+          async getSourceConfig() {
+            return {
+              tag: 'test_tag',
+              id: 'test_id',
+              type: 'test_type'
+            };
+          },
+          async getConnectionSchema() {
+            return [];
+          },
+          async getConnectionStatus() {
+            return {
+              id: 'test_id',
+              uri: 'http://example.org/',
+              connected: true,
+              errors: []
+            };
+          },
+          async getDebugTablesInfo() {
+            return [];
           }
-        } as Partial<RouteAPI>;
+        } satisfies Partial<RouteAPI> as unknown as RouteAPI;
       },
       addStopHandler() {
         return () => {};

package/test/src/routes/stream.test.ts

@@ -1,4 +1,4 @@
-import { BasicRouterRequest, Context, SyncRulesBucketStorage } from '@/index.js';
+import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
 import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { Readable, Writable } from 'stream';
@@ -8,6 +8,7 @@ import winston from 'winston';
 import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
 import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
 import { mockServiceContext } from './mocks.js';
+import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
 
 describe('Stream Route', () => {
   describe('compressed stream', () => {
@@ -15,11 +16,11 @@ describe('Stream Route', () => {
       const context: Context = {
         logger: logger,
         service_context: mockServiceContext(null),
-        token_payload: {
+        token_payload: new JwtPayload({
           sub: '',
           exp: 0,
           iat: 0
-        }
+        })
       };
 
       const request: BasicRouterRequest = {
@@ -45,7 +46,7 @@ describe('Stream Route', () => {
 
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}').hydrate();
+          return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');
@@ -56,11 +57,11 @@ describe('Stream Route', () => {
       const context: Context = {
         logger: logger,
         service_context: serviceContext,
-        token_payload: {
+        token_payload: new JwtPayload({
           exp: new Date().getTime() / 1000 + 10000,
           iat: new Date().getTime() / 1000 - 10000,
           sub: 'test-user'
-        }
+        })
       };
 
       // It may be worth eventually doing this via Fastify to test the full stack
@@ -83,7 +84,7 @@ describe('Stream Route', () => {
     it('logs the application metadata', async () => {
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}').hydrate();
+          return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');
@@ -108,11 +109,11 @@ describe('Stream Route', () => {
       const context: Context = {
         logger: testLogger,
         service_context: serviceContext,
-        token_payload: {
+        token_payload: new JwtPayload({
           exp: new Date().getTime() / 1000 + 10000,
           iat: new Date().getTime() / 1000 - 10000,
           sub: 'test-user'
-        }
+        })
       };
 
       const request: BasicRouterRequest = {