@powersync/service-core 0.0.0-dev-20250819134004 → 0.0.0-dev-20250825132649

package/src/routes/configure-fastify.ts

@@ -1,5 +1,4 @@
 import type fastify from 'fastify';
-import * as uuid from 'uuid';
 
 import { registerFastifyNotFoundHandler, registerFastifyRoutes } from './route-register.js';
 

package/src/routes/endpoints/socket-route.ts

@@ -23,10 +23,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       };
 
       const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
-        client_id: params.client_id,
+        client_id: params.client_id ?? '',
         user_id: context.user_id!,
         user_agent: context.user_agent,
-        jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp * 1000) : undefined,
+        // At this point the token_payload is guaranteed to be present
+        jwt_exp: new Date(context.token_payload!.exp * 1000),
         connected_at: new Date(streamStart)
       };
 
@@ -98,8 +99,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
           bucketStorage: bucketStorage,
           syncRules: syncRules,
           params: {
-            ...params,
-            binary_data: true // always true for web sockets
+            ...params
           },
           token: context!.token_payload!,
           tokenStreamOptions: {
@@ -108,7 +108,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
           },
           tracker,
           signal,
-          logger
+          logger,
+          isEncodingAsBson: true
         })) {
           if (signal.aborted) {
             break;

package/src/routes/endpoints/sync-stream.ts

@@ -30,9 +30,9 @@ export const syncStreamed = routeDefinition({
     const clientId = payload.params.client_id;
     const streamStart = Date.now();
     // This falls back to JSON unless there's preference for the bson-stream in the Accept header.
-    const useBson =
-      payload.request.headers.accept &&
-      new Negotiator(payload.request).mediaType(supportedContentTypes) == concatenatedBsonContentType;
+    const useBson = payload.request.headers.accept
+      ? new Negotiator(payload.request).mediaType(supportedContentTypes) == concatenatedBsonContentType
+      : false;
 
     logger.defaultMeta = {
       ...logger.defaultMeta,
@@ -42,10 +42,11 @@ export const syncStreamed = routeDefinition({
       bson: useBson
     };
     const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
-      client_id: clientId,
+      client_id: clientId ?? '',
       user_id: payload.context.user_id!,
       user_agent: userAgent as string,
-      jwt_exp: token_payload?.exp ? new Date(token_payload?.exp * 1000) : undefined,
+      // At this point the token_payload is guaranteed to be present
+      jwt_exp: new Date(token_payload!.exp * 1000),
       connected_at: new Date(streamStart)
     };
 
@@ -82,7 +83,8 @@ export const syncStreamed = routeDefinition({
         token: payload.context.token_payload!,
         tracker,
         signal: controller.signal,
-        logger
+        logger,
+        isEncodingAsBson: useBson
       });
 
       const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);

package/src/routes/router.ts

@@ -1,4 +1,4 @@
-import { router, ServiceError, Logger } from '@powersync/lib-services-framework';
+import { Logger, router } from '@powersync/lib-services-framework';
 import type { JwtPayload } from '../auth/auth-index.js';
 import { ServiceContext } from '../system/ServiceContext.js';
 import { RouterEngine } from './RouterEngine.js';
@@ -31,11 +31,11 @@ export type BasicRouterRequest = {
   hostname: string;
 };
 
-export type ConextProviderOptions = {
+export type ContextProviderOptions = {
   logger: Logger;
 };
 
-export type ContextProvider = (request: BasicRouterRequest, options: ConextProviderOptions) => Promise<Context>;
+export type ContextProvider = (request: BasicRouterRequest, options: ContextProviderOptions) => Promise<Context>;
 
 export type RequestEndpoint<
   I,

package/src/storage/ReportStorage.ts

@@ -1,11 +1,39 @@
 import { event_types } from '@powersync/service-types';
 
+/**
+ * Represents a configured report storage.
+ *
+ * Report storage is used for storing localized data for the instance.
+ * Data can then be used for reporting purposes.
+ *
+ */
 export interface ReportStorage extends AsyncDisposable {
+  /**
+   * Report a client connection.
+   */
   reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void>;
+  /**
+   * Report a client disconnection.
+   */
   reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void>;
-  getConnectedClients(data: event_types.ClientConnectionsRequest): Promise<event_types.ClientConnectionReport>;
+  /**
+   * Get currently connected clients.
+   * This will return any short or long term connected clients.
+   * Clients that have no disconnected_at timestamp and that have a valid jwt_exp timestamp are considered connected.
+   */
+  getConnectedClients(): Promise<event_types.ClientConnectionReportResponse>;
+  /**
+   * Get a report of client connections over a day, week or month.
+   * This is internally used to generate reports over it always returns the previous day, week or month.
+   * Usually this is call on the start of the new day, week or month. It will return all unique completed connections
+   * as well as uniques currently connected clients.
+   */
   getClientConnectionReports(
     data: event_types.ClientConnectionReportRequest
-  ): Promise<event_types.ClientConnectionReport>;
+  ): Promise<event_types.ClientConnectionReportResponse>;
+  /**
+   * Delete old connection data based on a specific date.
+   * This is used to clean up old connection data that is no longer needed.
+   */
   deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void>;
 }

package/src/sync/sync.ts

@@ -27,6 +27,7 @@ export interface SyncStreamParameters {
   params: util.StreamingSyncRequest;
   token: auth.JwtPayload;
   logger?: Logger;
+  isEncodingAsBson: boolean;
   /**
    * If this signal is aborted, the stream response ends as soon as possible, without error.
    */
@@ -39,7 +40,17 @@ export interface SyncStreamParameters {
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { syncContext, bucketStorage, syncRules, params, token, tokenStreamOptions, tracker, signal } = options;
+  const {
+    syncContext,
+    bucketStorage,
+    syncRules,
+    params,
+    token,
+    tokenStreamOptions,
+    tracker,
+    signal,
+    isEncodingAsBson
+  } = options;
   const logger = options.logger ?? defaultLogger;
 
   // We also need to be able to abort, so we create our own controller.
@@ -65,7 +76,8 @@ export async function* streamResponse(
     token,
     tracker,
     controller.signal,
-    logger
+    logger,
+    isEncodingAsBson
   );
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
@@ -93,9 +105,10 @@ async function* streamResponseInner(
   tokenPayload: RequestJwtPayload,
   tracker: RequestTracker,
   signal: AbortSignal,
-  logger: Logger
+  logger: Logger,
+  isEncodingAsBson: boolean
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
-  const { raw_data, binary_data } = params;
+  const { raw_data } = params;
 
   const userId = tokenPayload.sub;
   const checkpointUserId = util.checkpointUserId(userId as string, params.client_id);
@@ -225,8 +238,7 @@ async function* streamResponseInner(
           checkpoint: next.value.value.checkpoint,
           bucketsToFetch: buckets,
           checkpointLine: line,
-          raw_data,
-          binary_data,
+          legacyDataLines: !isEncodingAsBson && params.raw_data != true,
           onRowsSent: markOperationsSent,
           abort_connection: signal,
           abort_batch: abortCheckpointSignal,
@@ -255,8 +267,8 @@ interface BucketDataRequest {
   checkpointLine: CheckpointLine;
   /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
   bucketsToFetch: BucketDescription[];
-  raw_data: boolean | undefined;
-  binary_data: boolean | undefined;
+  /** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
+  legacyDataLines: boolean;
   /** Signals that the connection was aborted and that streaming should stop ASAP. */
   abort_connection: AbortSignal;
   /**
@@ -317,8 +329,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     checkpoint,
     bucketsToFetch,
     checkpointLine,
-    raw_data,
-    binary_data,
+    legacyDataLines,
     abort_connection,
     abort_batch,
     onRowsSent,
@@ -368,32 +379,21 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       }
       logger.debug(`Sending data for ${r.bucket}`);
 
-      let send_data: any;
-      if (binary_data) {
-        // Send the object as is, will most likely be encoded as a BSON document
-        send_data = { data: r };
-      } else if (raw_data) {
-        /**
-         * Data is a raw string - we can use the more efficient JSON.stringify.
-         */
-        const response: util.StreamingSyncData = {
-          data: r
-        };
-        send_data = JSON.stringify(response);
-      } else {
-        // We need to preserve the embedded data exactly, so this uses a JsonContainer
-        // and JSONBig to stringify.
-        const response: util.StreamingSyncData = {
-          data: transformLegacyResponse(r)
-        };
-        send_data = JSONBig.stringify(response);
-      }
-      yield { data: send_data, done: false };
-      if (send_data.length > 50_000) {
-        // IMPORTANT: This does not affect the output stream, but is used to flush
-        // iterator memory in case if large data sent.
-        yield { data: null, done: false };
-      }
+      const line = legacyDataLines
+        ? // We need to preserve the embedded data exactly, so this uses a JsonContainer
+          // and JSONBig to stringify.
+          JSONBig.stringify({
+            data: transformLegacyResponse(r)
+          } satisfies util.StreamingSyncData)
+        : // We can send the object as-is, which will be converted to JSON or BSON by a downstream transformer.
+          ({ data: r } satisfies util.StreamingSyncData);
+
+      yield { data: line, done: false };
+
+      // IMPORTANT: This does not affect the output stream, but is used to flush
+      // iterator memory in case if large data sent.
+      yield { data: null, done: false };
+
       onRowsSent(statsForBatch(r));
 
       checkpointLine.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });

package/src/util/protocol-types.ts

@@ -76,11 +76,6 @@ export const StreamingSyncRequest = t.object({
    */
   raw_data: t.boolean.optional(),
 
-  /**
-   * Data is received in a serialized BSON Buffer
-   */
-  binary_data: t.boolean.optional(),
-
   /**
    * Client parameters to be passed to the sync rules.
    */

package/test/src/sync/BucketChecksumState.test.ts

@@ -607,11 +607,20 @@ bucket_definitions:
   describe('streams', () => {
     let source: { -readonly [P in keyof BucketSource]: BucketSource[P] };
     let storage: MockBucketChecksumStateStorage;
-    let staticBucketIds = ['stream|0[]'];
 
-    function checksumState(options?: Partial<BucketChecksumStateOptions>) {
-      const rules = new SqlSyncRules('');
-      rules.bucketSources.push(source);
+    function checksumState(source: string | boolean, options?: Partial<BucketChecksumStateOptions>) {
+      if (typeof source == 'boolean') {
+        source = `
+streams:
+  stream:
+    auto_subscribe: ${source}
+    query: SELECT * FROM assets WHERE id IN ifnull(subscription.parameter('ids'), '["default"]');
+`;
+      }
+
+      const rules = SqlSyncRules.fromYaml(source, {
+        defaultSchema: 'public'
+      });
 
       return new BucketChecksumState({
         syncContext,
@@ -623,77 +632,15 @@ bucket_definitions:
       });
     }
 
-    function createQuerier(ids: string[], subscription: number | null): BucketParameterQuerier {
-      return {
-        staticBuckets: ids.map((bucket) => ({
-          definition: 'stream',
-          inclusion_reasons: subscription == null ? ['default'] : [{ subscription }],
-          bucket,
-          priority: 3
-        })),
-        hasDynamicBuckets: false,
-        parameterQueryLookups: [],
-        queryDynamicBucketDescriptions: function (): never {
-          throw new Error('no dynamic buckets.');
-        }
-      };
-    }
-
     beforeEach(() => {
-      // Currently using mocked streams before streams are actually implemented as parsable rules.
-      source = {
-        name: 'stream',
-        type: BucketSourceType.SYNC_STREAM,
-        subscribedToByDefault: false,
-        pushBucketParameterQueriers(result, options) {
-          // Create a fake querier that resolves the global stream["default"] bucket by default and allows extracting
-          // additional buckets from parameters.
-          const subscriptions = options.streams['stream'] ?? [];
-          if (!this.subscribedToByDefault && !subscriptions.length) {
-            return;
-          }
-
-          let hasExplicitDefaultSubscription = false;
-          for (const subscription of subscriptions) {
-            try {
-              let subscriptionParameters = [];
-
-              if (subscription.parameters != null) {
-                subscriptionParameters = JSON.parse(subscription.parameters['ids'] as string).map(
-                  (e: string) => `stream["${e}"]`
-                );
-              } else {
-                hasExplicitDefaultSubscription = true;
-              }
-
-              result.queriers.push(createQuerier([...subscriptionParameters], subscription.opaque_id));
-            } catch (e) {
-              result.errors.push({
-                descriptor: 'stream',
-                subscription,
-                message: `Error evaluating bucket ids: ${e.message}`
-              });
-            }
-          }
-
-          // If the stream is subscribed to by default and there is no explicit subscription that would match the default
-          // subscription, also include the default querier.
-          if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) {
-            result.queriers.push(createQuerier(['stream["default"]'], null));
-          }
-        }
-      } satisfies Partial<BucketSource> as any;
-
       storage = new MockBucketChecksumStateStorage();
-      storage.updateTestChecksum({ bucket: 'stream["default"]', checksum: 1, count: 1 });
-      storage.updateTestChecksum({ bucket: 'stream["a"]', checksum: 1, count: 1 });
-      storage.updateTestChecksum({ bucket: 'stream["b"]', checksum: 1, count: 1 });
+      storage.updateTestChecksum({ bucket: 'stream|0["default"]', checksum: 1, count: 1 });
+      storage.updateTestChecksum({ bucket: 'stream|0["a"]', checksum: 1, count: 1 });
+      storage.updateTestChecksum({ bucket: 'stream|0["b"]', checksum: 1, count: 1 });
     });
 
     test('includes defaults', async () => {
-      source.subscribedToByDefault = true;
-      const state = checksumState();
-
+      const state = checksumState(true);
       const line = await state.buildNextCheckpointLine({
         base: storage.makeCheckpoint(1n),
         writeCheckpoint: null,
@@ -703,7 +650,7 @@ bucket_definitions:
       expect(line?.checkpointLine).toEqual({
         checkpoint: {
           buckets: [
-            { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
+            { bucket: 'stream|0["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
           ],
           last_op_id: '1',
           write_checkpoint: undefined,
@@ -713,8 +660,7 @@ bucket_definitions:
     });
 
     test('can exclude defaults', async () => {
-      source.subscribedToByDefault = true;
-      const state = checksumState({ syncRequest: { streams: { include_defaults: false, subscriptions: [] } } });
+      const state = checksumState(true, { syncRequest: { streams: { include_defaults: false, subscriptions: [] } } });
 
       const line = await state.buildNextCheckpointLine({
         base: storage.makeCheckpoint(1n),
@@ -733,9 +679,7 @@ bucket_definitions:
     });
 
     test('custom subscriptions', async () => {
-      source.subscribedToByDefault = true;
-
-      const state = checksumState({
+      const state = checksumState(true, {
         syncRequest: {
           streams: {
             subscriptions: [
@@ -755,9 +699,9 @@ bucket_definitions:
       expect(line?.checkpointLine).toEqual({
         checkpoint: {
           buckets: [
-            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
-            { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 1 }] },
-            { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
+            { bucket: 'stream|0["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 1 }] },
+            { bucket: 'stream|0["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
           ],
           last_op_id: '1',
           write_checkpoint: undefined,
@@ -767,7 +711,7 @@ bucket_definitions:
     });
 
     test('overlap between custom subscriptions', async () => {
-      const state = checksumState({
+      const state = checksumState(false, {
         syncRequest: {
           streams: {
             subscriptions: [
@@ -787,8 +731,8 @@ bucket_definitions:
       expect(line?.checkpointLine).toEqual({
         checkpoint: {
           buckets: [
-            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
-            { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }, { sub: 1 }] }
+            { bucket: 'stream|0["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }, { sub: 1 }] }
           ],
           last_op_id: '1',
           write_checkpoint: undefined,
@@ -798,8 +742,7 @@ bucket_definitions:
     });
 
     test('overlap between default and custom subscription', async () => {
-      source.subscribedToByDefault = true;
-      const state = checksumState({
+      const state = checksumState(true, {
         syncRequest: {
           streams: {
             subscriptions: [{ stream: 'stream', parameters: { ids: '["a", "default"]' }, override_priority: 1 }]
@@ -816,9 +759,9 @@ bucket_definitions:
       expect(line?.checkpointLine).toEqual({
         checkpoint: {
           buckets: [
-            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream|0["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
             {
-              bucket: 'stream["default"]',
+              bucket: 'stream|0["default"]',
               checksum: 1,
               count: 1,
               priority: 1,
@@ -833,9 +776,7 @@ bucket_definitions:
     });
 
     test('reports errors', async () => {
-      source.subscribedToByDefault = true;
-
-      const state = checksumState({
+      const state = checksumState(true, {
         syncRequest: {
           streams: {
             subscriptions: [
@@ -855,10 +796,10 @@ bucket_definitions:
       expect(line?.checkpointLine).toEqual({
         checkpoint: {
           buckets: [
-            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
-            { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream|0["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
             {
-              bucket: 'stream["default"]',
+              bucket: 'stream|0["default"]',
               checksum: 1,
               count: 1,
               priority: 3,