@powersync/service-core 1.13.4 → 1.15.0

package/src/sync/sync.ts

@@ -1,5 +1,11 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import { BucketDescription, BucketPriority, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
+import {
+  BucketDescription,
+  BucketPriority,
+  RequestJwtPayload,
+  RequestParameters,
+  SqlSyncRules
+} from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -8,7 +14,7 @@ import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
-import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
+import { BucketChecksumState, CheckpointLine, VersionedSyncRules } from './BucketChecksumState.js';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
 import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
 import { SyncContext } from './SyncContext.js';
@@ -17,11 +23,11 @@ import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTra
 export interface SyncStreamParameters {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
-  syncRules: SqlSyncRules;
+  syncRules: VersionedSyncRules;
   params: util.StreamingSyncRequest;
-  syncParams: RequestParameters;
   token: auth.JwtPayload;
   logger?: Logger;
+  isEncodingAsBson: boolean;
   /**
    * If this signal is aborted, the stream response ends as soon as possible, without error.
    */
@@ -34,8 +40,17 @@ export interface SyncStreamParameters {
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { syncContext, bucketStorage, syncRules, params, syncParams, token, tokenStreamOptions, tracker, signal } =
-    options;
+  const {
+    syncContext,
+    bucketStorage,
+    syncRules,
+    params,
+    token,
+    tokenStreamOptions,
+    tracker,
+    signal,
+    isEncodingAsBson
+  } = options;
   const logger = options.logger ?? defaultLogger;
 
   // We also need to be able to abort, so we create our own controller.
@@ -58,10 +73,11 @@ export async function* streamResponse(
     bucketStorage,
     syncRules,
     params,
-    syncParams,
+    token,
     tracker,
     controller.signal,
-    logger
+    logger,
+    isEncodingAsBson
   );
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
@@ -84,26 +100,25 @@ export async function* streamResponse(
 async function* streamResponseInner(
   syncContext: SyncContext,
   bucketStorage: storage.SyncRulesBucketStorage,
-  syncRules: SqlSyncRules,
+  syncRules: VersionedSyncRules,
   params: util.StreamingSyncRequest,
-  syncParams: RequestParameters,
+  tokenPayload: RequestJwtPayload,
   tracker: RequestTracker,
   signal: AbortSignal,
-  logger: Logger
+  logger: Logger,
+  isEncodingAsBson: boolean
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
-  const { raw_data, binary_data } = params;
+  const { raw_data } = params;
 
-  const checkpointUserId = util.checkpointUserId(syncParams.tokenParameters.user_id as string, params.client_id);
+  const userId = tokenPayload.sub;
+  const checkpointUserId = util.checkpointUserId(userId as string, params.client_id);
 
   const checksumState = new BucketChecksumState({
     syncContext,
     bucketStorage,
     syncRules,
-    syncParams,
-    initialBucketPositions: params.buckets?.map((bucket) => ({
-      name: bucket.name,
-      after: BigInt(bucket.after)
-    })),
+    tokenPayload,
+    syncRequest: params,
     logger: logger
   });
   const stream = bucketStorage.watchCheckpointChanges({
@@ -223,12 +238,11 @@ async function* streamResponseInner(
           checkpoint: next.value.value.checkpoint,
           bucketsToFetch: buckets,
           checkpointLine: line,
-          raw_data,
-          binary_data,
+          legacyDataLines: !isEncodingAsBson && params.raw_data != true,
           onRowsSent: markOperationsSent,
           abort_connection: signal,
           abort_batch: abortCheckpointSignal,
-          user_id: syncParams.userId,
+          user_id: userId,
           // Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial
           // sync complete message instead.
           forPriority: !isLast ? priority : null,
@@ -253,8 +267,8 @@ interface BucketDataRequest {
   checkpointLine: CheckpointLine;
   /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
   bucketsToFetch: BucketDescription[];
-  raw_data: boolean | undefined;
-  binary_data: boolean | undefined;
+  /** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
+  legacyDataLines: boolean;
   /** Signals that the connection was aborted and that streaming should stop ASAP. */
   abort_connection: AbortSignal;
   /**
@@ -315,8 +329,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     checkpoint,
     bucketsToFetch,
     checkpointLine,
-    raw_data,
-    binary_data,
+    legacyDataLines,
     abort_connection,
     abort_batch,
     onRowsSent,
@@ -366,32 +379,21 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       }
       logger.debug(`Sending data for ${r.bucket}`);
 
-      let send_data: any;
-      if (binary_data) {
-        // Send the object as is, will most likely be encoded as a BSON document
-        send_data = { data: r };
-      } else if (raw_data) {
-        /**
-         * Data is a raw string - we can use the more efficient JSON.stringify.
-         */
-        const response: util.StreamingSyncData = {
-          data: r
-        };
-        send_data = JSON.stringify(response);
-      } else {
-        // We need to preserve the embedded data exactly, so this uses a JsonContainer
-        // and JSONBig to stringify.
-        const response: util.StreamingSyncData = {
-          data: transformLegacyResponse(r)
-        };
-        send_data = JSONBig.stringify(response);
-      }
-      yield { data: send_data, done: false };
-      if (send_data.length > 50_000) {
-        // IMPORTANT: This does not affect the output stream, but is used to flush
-        // iterator memory in case if large data sent.
-        yield { data: null, done: false };
-      }
+      const line = legacyDataLines
+        ? // We need to preserve the embedded data exactly, so this uses a JsonContainer
+          // and JSONBig to stringify.
+          JSONBig.stringify({
+            data: transformLegacyResponse(r)
+          } satisfies util.StreamingSyncData)
+        : // We can send the object as-is, which will be converted to JSON or BSON by a downstream transformer.
+          ({ data: r } satisfies util.StreamingSyncData);
+
+      yield { data: line, done: false };
+
+      // IMPORTANT: This does not affect the output stream, but is used to flush
+      // iterator memory in case if large data sent.
+      yield { data: null, done: false };
+
       onRowsSent(statsForBatch(r));
 
       checkpointLine.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });

package/src/sync/util.ts

@@ -3,6 +3,7 @@ import * as timers from 'timers/promises';
 import { SemaphoreInterface } from 'async-mutex';
 import * as util from '../util/util-index.js';
 import { RequestTracker } from './RequestTracker.js';
+import { serialize } from 'bson';
 
 export type TokenStreamOptions = {
   /**
@@ -76,6 +77,27 @@ export async function* tokenStream(
   }
 }
 
+export function syncLineToBson(line: string | Record<string, any>): Buffer {
+  if (typeof line == 'string') {
+    // Should not happen with binary_data: true
+    throw new Error(`Unexpected string data: ${line}`);
+  } else {
+    // On NodeJS, serialize always returns a Buffer
+    return serialize(line) as Buffer;
+  }
+}
+
+export async function* bsonLines(iterator: AsyncIterable<string | null | Record<string, any>>): AsyncGenerator<Buffer> {
+  for await (let line of iterator) {
+    if (line == null) {
+      // Empty value just to flush iterator memory
+      continue;
+    } else {
+      yield syncLineToBson(line);
+    }
+  }
+}
+
 export async function* ndjson(iterator: AsyncIterable<string | null | Record<string, any>>): AsyncGenerator<string> {
   for await (let data of iterator) {
     if (data == null) {
@@ -91,12 +113,19 @@ export async function* ndjson(iterator: AsyncIterable<string | null | Record<str
 }
 
 export async function* transformToBytesTracked(
-  iterator: AsyncIterable<string>,
+  iterator: AsyncIterable<string | Buffer>,
   tracker: RequestTracker
 ): AsyncGenerator<Buffer> {
   for await (let data of iterator) {
-    const encoded = Buffer.from(data, 'utf8');
-    tracker.addDataSynced(encoded.length);
+    let encoded: Buffer;
+
+    if (typeof data == 'string') {
+      encoded = Buffer.from(data, 'utf8');
+    } else {
+      encoded = data;
+    }
+
+    tracker.addPlaintextDataSynced(encoded.length);
     yield encoded;
   }
 }

package/src/util/config/compound-config-collector.ts

@@ -89,6 +89,7 @@ export class CompoundConfigCollector {
           }
         ])
       );
+      keyStore.supabaseAuthDebug.sharedSecretEnabled = true;
     }
 
     let jwks_uris = baseConfig.client_auth?.jwks_uri ?? [];
@@ -114,6 +115,29 @@ export class CompoundConfigCollector {
     for (let uri of jwks_uris) {
       collectors.add(new auth.CachedKeyCollector(new auth.RemoteJWKSCollector(uri, { lookupOptions: jwksLookup })));
     }
+    const supabaseAuthDetails = auth.getSupabaseJwksUrl(baseConfig.replication?.connections?.[0]);
+    keyStore.supabaseAuthDebug.jwksDetails = supabaseAuthDetails;
+
+    if (baseConfig.client_auth?.supabase) {
+      // Automatic support for Supabase signing keys:
+      // https://supabase.com/docs/guides/auth/signing-keys
+      if (supabaseAuthDetails != null) {
+        const collector = new auth.RemoteJWKSCollector(supabaseAuthDetails.url, {
+          lookupOptions: jwksLookup,
+          // Special case aud and max lifetime for Supabase keys
+          keyOptions: auth.SUPABASE_KEY_OPTIONS
+        });
+        collectors.add(new auth.CachedKeyCollector(collector));
+        keyStore.supabaseAuthDebug.jwksEnabled = true;
+        logger.info(`Configured Supabase Auth with ${supabaseAuthDetails.url}`);
+      } else {
+        logger.warn(
+          'Supabase Auth is enabled, but no Supabase connection string found. Skipping Supabase JWKS URL configuration.'
+        );
+      }
+    } else if (supabaseAuthDetails != null) {
+      logger.warn(`Supabase connection string found, but Supabase Auth is not enabled in the config.`);
+    }
 
     const sync_rules = await this.collectSyncRules(baseConfig, runnerConfig);
 

package/src/util/lsn.ts

@@ -0,0 +1,8 @@
+/**
+ * Return the larger of two LSNs.
+ */
+export function maxLsn(a: string | null | undefined, b: string | null | undefined): string | null {
+  if (a == null) return b ?? null;
+  if (b == null) return a;
+  return a > b ? a : b;
+}

package/src/util/protocol-types.ts

@@ -13,9 +13,51 @@ export const BucketRequest = t.object({
 
 export type BucketRequest = t.Decoded<typeof BucketRequest>;
 
+/**
+ * A sync steam that a client has expressed interest in by explicitly opening it on the client side.
+ */
+export const RequestedStreamSubscription = t.object({
+  /**
+   * The defined name of the stream as it appears in sync stream definitions.
+   */
+  stream: t.string,
+  /**
+   * An optional dictionary of parameters to pass to this specific stream.
+   */
+  parameters: t.record(t.any).optional(),
+  /**
+   * Set when the client wishes to re-assign a different priority to this stream.
+   *
+   * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
+   * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
+   */
+  override_priority: t.union(t.number, t.Null)
+});
+
+export type RequestedStreamSubscription = t.Decoded<typeof RequestedStreamSubscription>;
+
+/**
+ * An overview of all subscribed streams as part of a streaming sync request.
+ */
+export const StreamSubscriptionRequest = t.object({
+  /**
+   * Whether to sync default streams.
+   *
+   * When disabled, only explicitly-opened subscriptions are included.
+   */
+  include_defaults: t.boolean.optional(),
+
+  /**
+   * An array of sync streams the client has opened explicitly.
+   */
+  subscriptions: t.array(RequestedStreamSubscription)
+});
+
+export type StreamSubscriptionRequest = t.Decoded<typeof StreamSubscriptionRequest>;
+
 export const StreamingSyncRequest = t.object({
   /**
-   * Existing bucket states.
+   * Existing client-side bucket states.
    */
   buckets: t.array(BucketRequest).optional(),
 
@@ -34,11 +76,6 @@ export const StreamingSyncRequest = t.object({
    */
   raw_data: t.boolean.optional(),
 
-  /**
-   * Data is received in a serialized BSON Buffer
-   */
-  binary_data: t.boolean.optional(),
-
   /**
    * Client parameters to be passed to the sync rules.
    */
@@ -47,7 +84,12 @@ export const StreamingSyncRequest = t.object({
   /**
    * Unique client id.
    */
-  client_id: t.string.optional()
+  client_id: t.string.optional(),
+
+  /**
+   * If the client is aware of streams, an array of streams the client has opened.
+   */
+  streams: StreamSubscriptionRequest.optional()
 });
 
 export type StreamingSyncRequest = t.Decoded<typeof StreamingSyncRequest>;
@@ -60,7 +102,7 @@ export interface StreamingSyncCheckpointDiff {
   checkpoint_diff: {
     last_op_id: ProtocolOpId;
     write_checkpoint?: ProtocolOpId;
-    updated_buckets: BucketChecksumWithDescription[];
+    updated_buckets: CheckpointBucket[];
     removed_buckets: string[];
   };
 }
@@ -99,10 +141,54 @@ export type StreamingSyncLine =
  */
 export type ProtocolOpId = string;
 
+export interface StreamDescription {
+  /**
+   * The name of the stream as it appears in the sync configuration.
+   */
+  name: string;
+
+  /**
+   * Whether this stream is subscribed to by default.
+   *
+   * For default streams, this field is still `true` if clients have an explicit subscription to the stream.
+   */
+  is_default: boolean;
+
+  /**
+   * If some subscriptions on this stream could not be resolved, e.g. due to an error, this array contains the faulty
+   * subscriptions along with an error message.
+   */
+  errors: StreamSubscriptionError[];
+}
+
+export interface StreamSubscriptionError {
+  /**
+   * The subscription that errored - either the default subscription or some of the explicit subscriptions.
+   */
+  subscription: 'default' | number;
+  /**
+   * A message describing the error on the subscription.
+   */
+  message: string;
+}
+
 export interface Checkpoint {
   last_op_id: ProtocolOpId;
   write_checkpoint?: ProtocolOpId;
-  buckets: BucketChecksumWithDescription[];
+  buckets: CheckpointBucket[];
+
+  /**
+   * All streams that the client is subscribed to.
+   *
+   * This field has two purposes:
+   *
+   *  1. It allows clients to determine which of their subscriptions actually works. E.g. if a user does
+   *     `db.syncStream('non_existent_stream').subscribe()`, clients don't immediately know that the stream doesn't
+   *     exist. Only after the next `checkpoint` line can they query this field and mark unresolved subscriptions.
+   *. 2. It allows clients to learn which default streams they have been subscribed to. This is relevant for APIs
+   *     listing all streams on the client-side.
+   */
+  streams: StreamDescription[];
 }
 
 export interface BucketState {
@@ -158,4 +244,46 @@ export interface BucketChecksum {
   count: number;
 }
 
-export interface BucketChecksumWithDescription extends BucketChecksum, BucketDescription {}
+/**
+ * The reason a particular bucket is included in a checkpoint.
+ *
+ * This information allows clients to associate individual buckets with sync streams they're subscribed to. Having that
+ * association is useful because it enables clients to track progress for individual sync streams.
+ */
+export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription;
+
+/**
+ * A bucket has been included in a checkpoint because it's part of a default stream.
+ */
+export type BucketDerivedFromDefaultStream = {
+  /**
+   * The index (into {@link Checkpoint.streams}) of the stream defining the bucket.
+   */
+  default: number;
+};
+
+/**
+ * The bucket has been included in a checkpoint because it's part of a stream that a client has explicitly subscribed
+ * to.
+ */
+export type BucketDerivedFromExplicitSubscription = {
+  /**
+   * The index (into {@link StreamSubscriptionRequest.subscriptions}) of the subscription yielding this bucket.
+   */
+  sub: number;
+};
+
+export interface ClientBucketDescription {
+  /**
+   * An opaque id of the bucket.
+   */
+  bucket: string;
+  /**
+   * The priority used to synchronize this bucket, derived from its definition and an optional priority override from
+   * the stream subscription.
+   */
+  priority: BucketPriority;
+  subscriptions: BucketSubscriptionReason[];
+}
+
+export interface CheckpointBucket extends BucketChecksum, ClientBucketDescription {}

package/src/util/util-index.ts

@@ -1,5 +1,6 @@
 export * from './alerting.js';
 export * from './env.js';
+export * from './lsn.js';
 export * from './memory-tracking.js';
 export * from './Mutex.js';
 export * from './protocol-types.js';

package/src/util/utils.ts

@@ -6,11 +6,26 @@ import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
-import { PartialChecksum } from '../storage/ChecksumCache.js';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+/**
+ * A partial checksum can never be used on its own - must always be combined with a full BucketChecksum.
+ */
+export interface PartialChecksum {
+  bucket: string;
+  /**
+   * 32-bit unsigned hash.
+   */
+  partialChecksum: number;
+
+  /**
+   * Count of operations - informational only.
+   */
+  partialCount: number;
+}
+
 /**
  * op_id as used internally, for individual operations and checkpoints.
  *
@@ -83,20 +98,49 @@ export function addChecksums(a: number, b: number) {
   return (a + b) & 0xffffffff;
 }
 
-export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | null): BucketChecksum {
-  if (b == null) {
-    return a;
-  } else if (b.isFullChecksum) {
+export function isPartialChecksum(c: PartialChecksum | BucketChecksum): c is PartialChecksum {
+  return 'partialChecksum' in c;
+}
+
+export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | BucketChecksum | null): BucketChecksum {
+  const checksum = addPartialChecksums(a.bucket, a, b);
+  if (isPartialChecksum(checksum)) {
+    // Should not happen since a != null
+    throw new ServiceAssertionError('Expected full checksum');
+  }
+  return checksum;
+}
+
+export function addPartialChecksums(
+  bucket: string,
+  a: BucketChecksum | null,
+  b: PartialChecksum | BucketChecksum | null
+): PartialChecksum | BucketChecksum {
+  if (a != null && b != null) {
+    if (!isPartialChecksum(b)) {
+      // Replaces preState
+      return b;
+    }
+    // merge
+    return {
+      bucket,
+      checksum: addChecksums(a.checksum, b.partialChecksum),
+      count: a.count + b.partialCount
+    };
+  } else if (a != null) {
     return {
-      bucket: b.bucket,
-      count: b.partialCount,
-      checksum: b.partialChecksum
+      bucket,
+      checksum: a.checksum,
+      count: a.count
     };
+  } else if (b != null) {
+    return b;
   } else {
+    // No data found (may still have a previously-cached checksum).
     return {
-      bucket: a.bucket,
-      count: a.count + b.partialCount,
-      checksum: addChecksums(a.checksum, b.partialChecksum)
+      bucket,
+      partialChecksum: 0,
+      partialCount: 0
     };
   }
 }
@@ -148,7 +192,10 @@ export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
  *
  * If we don't store data, we assume we always have a complete row.
  */
-export function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
+export function isCompleteRow(
+  storeData: boolean,
+  row: sync_rules.ToastableSqliteRow
+): row is sync_rules.SqliteInputRow {
   if (!storeData) {
     // Assume the row is complete - no need to check
     return true;