@powersync/service-core 0.0.0-dev-20251111070830 → 0.0.0-dev-20251120150014

package/src/routes/endpoints/sync-stream.ts

@@ -5,10 +5,11 @@ import { Readable } from 'stream';
 import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 
+import { APIMetric, event_types } from '@powersync/service-types';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
-import { APIMetric, event_types } from '@powersync/service-types';
 
+import { formatParamsForLogging } from '../../util/param-logging.js';
 import { maybeCompressResponseStream } from '../compression.js';
 
 export enum SyncRoutes {
@@ -42,7 +43,10 @@ export const syncStreamed = routeDefinition({
       user_agent: userAgent,
       client_id: clientId,
       user_id: payload.context.user_id,
-      bson: useBson
+      bson: useBson,
+      app_metadata: payload.params.applicationMetadata
+        ? formatParamsForLogging(payload.params.applicationMetadata)
+        : undefined
     };
     const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
       client_id: clientId ?? '',
@@ -75,6 +79,7 @@ export const syncStreamed = routeDefinition({
 
     const controller = new AbortController();
     const tracker = new sync.RequestTracker(metricsEngine);
+
     try {
       metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
       service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData);
@@ -133,7 +138,7 @@ export const syncStreamed = routeDefinition({
       return new router.RouterResponse({
         status: 200,
         headers: {
-          'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType,
+          'Content-Type': 'text/event-stream', // useBson ? concatenatedBsonContentType : ndJsonContentType,
           ...encodingHeaders
         },
         data: stream,

package/src/storage/SyncRulesBucketStorage.ts

@@ -217,6 +217,11 @@ export interface CompactOptions {
   /** Minimum of 1 */
   moveBatchQueryLimit?: number;
 
+  /**
+   * Minimum of 1, default of 10.
+   */
+  minBucketChanges?: number;
+
   /**
    * Internal/testing use: Cache size for compacting parameters.
    */

package/src/sync/sync.ts

@@ -1,11 +1,5 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import {
-  BucketDescription,
-  BucketPriority,
-  RequestJwtPayload,
-  RequestParameters,
-  SqlSyncRules
-} from '@powersync/service-sync-rules';
+import { BucketDescription, BucketPriority, RequestJwtPayload } from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -14,11 +8,12 @@ import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
-import { BucketChecksumState, CheckpointLine, VersionedSyncRules } from './BucketChecksumState.js';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
-import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
-import { SyncContext } from './SyncContext.js';
+import { formatParamsForLogging } from '../util/param-logging.js';
+import { BucketChecksumState, CheckpointLine, VersionedSyncRules } from './BucketChecksumState.js';
 import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
+import { SyncContext } from './SyncContext.js';
+import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
 
 export interface SyncStreamParameters {
   syncContext: SyncContext;
@@ -53,6 +48,11 @@ export async function* streamResponse(
   } = options;
   const logger = options.logger ?? defaultLogger;
 
+  logger.info('Sync stream started', {
+    client_params: params.parameters ? formatParamsForLogging(params.parameters) : undefined,
+    streams: params.streams?.subscriptions.map((subscription) => subscription.stream)
+  });
+
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {

package/src/util/param-logging.ts

@@ -0,0 +1,40 @@
+export type ParamLoggingFormatOptions = {
+  maxKeyCount: number;
+  maxStringLength: number;
+};
+
+export const DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS: ParamLoggingFormatOptions = {
+  maxKeyCount: 10,
+  maxStringLength: 20
+};
+
+export function formatParamsForLogging(params: Record<string, any>, options: Partial<ParamLoggingFormatOptions> = {}) {
+  const {
+    maxStringLength = DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS.maxStringLength,
+    maxKeyCount = DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS.maxKeyCount
+  } = options;
+
+  function trimString(value: string): string {
+    if (value.length > maxStringLength) {
+      return value.slice(0, maxStringLength - 3) + '...';
+    }
+    return value;
+  }
+
+  return Object.fromEntries(
+    Object.entries(params).map(([key, value], index) => {
+      if (index == maxKeyCount) {
+        return ['[...]', '[...]'];
+      }
+
+      if (index > maxKeyCount) {
+        return [];
+      }
+
+      if (typeof value === 'string') {
+        return [key, trimString(value)];
+      }
+      return [key, trimString(JSON.stringify(value))];
+    })
+  );
+}

package/src/util/protocol-types.ts

@@ -1,6 +1,6 @@
-import * as t from 'ts-codec';
-import { BucketPriority, SqliteJsonRow } from '@powersync/service-sync-rules';
 import { JsonContainer } from '@powersync/service-jsonbig';
+import { BucketPriority, SqliteJsonRow } from '@powersync/service-sync-rules';
+import * as t from 'ts-codec';
 
 export const BucketRequest = t.object({
   name: t.string,
@@ -81,6 +81,11 @@ export const StreamingSyncRequest = t.object({
    */
   parameters: t.record(t.any).optional(),
 
+  /**
+   * Application metadata to be used in logging.
+   */
+  applicationMetadata: t.record(t.string).optional(),
+
   /**
    * Unique client id.
    */

package/test/src/routes/stream.test.ts

@@ -1,10 +1,12 @@
 import { BasicRouterRequest, Context, SyncRulesBucketStorage } from '@/index.js';
-import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
+import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { Readable, Writable } from 'stream';
 import { pipeline } from 'stream/promises';
 import { describe, expect, it } from 'vitest';
+import winston from 'winston';
 import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
+import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, formatParamsForLogging } from '../../../src/util/param-logging.js';
 import { mockServiceContext } from './mocks.js';
 
 describe('Stream Route', () => {
@@ -77,6 +79,88 @@ describe('Stream Route', () => {
       const r = await drainWithTimeout(stream).catch((error) => error);
       expect(r.message).toContain('Simulated storage error');
     });
+
+    it('logs the application metadata', async () => {
+      const storage = {
+        getParsedSyncRules() {
+          return new SqlSyncRules('bucket_definitions: {}');
+        },
+        watchCheckpointChanges: async function* (options) {
+          throw new Error('Simulated storage error');
+        }
+      } as Partial<SyncRulesBucketStorage>;
+      const serviceContext = mockServiceContext(storage);
+
+      // Create a custom format to capture log info objects (which include defaultMeta)
+      // Winston merges defaultMeta into the info object during formatting
+      const capturedLogs: any[] = [];
+      const captureFormat = winston.format((info) => {
+        // Capture the info object which includes defaultMeta merged in
+        capturedLogs.push({ ...info });
+        return info;
+      });
+
+      // Create a test logger with the capture format
+      const testLogger = winston.createLogger({
+        format: winston.format.combine(captureFormat(), winston.format.json()),
+        transports: [new winston.transports.Console()]
+      });
+
+      const context: Context = {
+        logger: testLogger,
+        service_context: serviceContext,
+        token_payload: {
+          exp: new Date().getTime() / 1000 + 10000,
+          iat: new Date().getTime() / 1000 - 10000,
+          sub: 'test-user'
+        }
+      };
+
+      const request: BasicRouterRequest = {
+        headers: {
+          'accept-encoding': 'gzip'
+        },
+        hostname: '',
+        protocol: 'http'
+      };
+
+      const inputMeta = {
+        test: 'test',
+        long_meta: 'a'.repeat(1000)
+      };
+
+      const response = await (syncStreamed.handler({
+        context,
+        params: {
+          applicationMetadata: inputMeta,
+          parameters: {
+            user_name: 'bob'
+          }
+        },
+        request
+      }) as Promise<RouterResponse>);
+      expect(response.status).toEqual(200);
+      const stream = response.data as Readable;
+      const r = await drainWithTimeout(stream).catch((error) => error);
+      expect(r.message).toContain('Simulated storage error');
+
+      // Find the "Sync stream started" log entry
+      const syncStartedLog = capturedLogs.find((log) => log.message === 'Sync stream started');
+      expect(syncStartedLog).toBeDefined();
+
+      // Verify that app_metadata from defaultMeta is present in the log
+      expect(syncStartedLog?.app_metadata).toBeDefined();
+      expect(syncStartedLog?.app_metadata).toEqual(formatParamsForLogging(inputMeta));
+      // Should trim long metadata
+      expect(syncStartedLog?.app_metadata.long_meta.length).toEqual(
+        DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS.maxStringLength
+      );
+
+      // Verify the explicit log parameters
+      expect(syncStartedLog?.client_params).toEqual({
+        user_name: 'bob'
+      });
+    });
   });
 });