@powersync/service-core 1.13.4 → 1.14.0

package/src/routes/endpoints/sync-stream.ts

@@ -1,6 +1,7 @@
 import { ErrorCode, errors, logger, router, schema } from '@powersync/lib-services-framework';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { Readable } from 'stream';
+import Negotiator from 'negotiator';
 
 import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
@@ -14,6 +15,10 @@ export enum SyncRoutes {
   STREAM = '/sync/stream'
 }
 
+const ndJsonContentType = 'application/x-ndjson';
+const concatenatedBsonContentType = 'application/vnd.powersync.bson-stream';
+const supportedContentTypes = [ndJsonContentType, concatenatedBsonContentType];
+
 export const syncStreamed = routeDefinition({
   path: SyncRoutes.STREAM,
   method: router.HTTPMethod.POST,
@@ -26,12 +31,17 @@ export const syncStreamed = routeDefinition({
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
     const clientId = payload.params.client_id;
     const streamStart = Date.now();
+    // This falls back to JSON unless there's preference for the bson-stream in the Accept header.
+    const useBson =
+      payload.request.headers.accept &&
+      new Negotiator(payload.request).mediaType(supportedContentTypes) == concatenatedBsonContentType;
 
     logger.defaultMeta = {
       ...logger.defaultMeta,
       user_agent: userAgent,
       client_id: clientId,
-      user_id: payload.context.user_id
+      user_id: payload.context.user_id,
+      bson: useBson
     };
 
     if (routerEngine.closed) {
@@ -61,25 +71,23 @@ export const syncStreamed = routeDefinition({
     const tracker = new sync.RequestTracker(metricsEngine);
     try {
       metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
-      const stream = Readable.from(
-        sync.transformToBytesTracked(
-          sync.ndjson(
-            sync.streamResponse({
-              syncContext: syncContext,
-              bucketStorage,
-              syncRules: syncRules,
-              params,
-              syncParams,
-              token: payload.context.token_payload!,
-              tracker,
-              signal: controller.signal,
-              logger
-            })
-          ),
-          tracker
-        ),
-        { objectMode: false, highWaterMark: 16 * 1024 }
-      );
+      const syncLines = sync.streamResponse({
+        syncContext: syncContext,
+        bucketStorage,
+        syncRules: syncRules,
+        params,
+        syncParams,
+        token: payload.context.token_payload!,
+        tracker,
+        signal: controller.signal,
+        logger
+      });
+
+      const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);
+      const stream = Readable.from(sync.transformToBytesTracked(byteContents, tracker), {
+        objectMode: false,
+        highWaterMark: 16 * 1024
+      });
 
       // Best effort guess on why the stream was closed.
       // We use the `??=` operator everywhere, so that we catch the first relevant
@@ -114,7 +122,7 @@ export const syncStreamed = routeDefinition({
       return new router.RouterResponse({
         status: 200,
         headers: {
-          'Content-Type': 'application/x-ndjson'
+          'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType
         },
         data: stream,
         afterSend: async (details) => {

package/src/routes/route-register.ts

@@ -1,8 +1,17 @@
 import type fastify from 'fastify';
 import * as uuid from 'uuid';
 
-import { errors, HTTPMethod, logger, router } from '@powersync/lib-services-framework';
+import {
+  ErrorCode,
+  errors,
+  HTTPMethod,
+  logger,
+  RouteNotFound,
+  router,
+  ServiceError
+} from '@powersync/lib-services-framework';
 import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
+import { FastifyReply } from 'fastify';
 
 export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {
   parse?: boolean;
@@ -69,23 +78,11 @@ export function registerFastifyRoutes(
             const serviceError = errors.asServiceError(ex);
             requestLogger.error(`Request failed`, serviceError);
 
-            response = new router.RouterResponse({
-              status: serviceError.errorData.status || 500,
-              headers: {
-                'Content-Type': 'application/json'
-              },
-              data: {
-                error: serviceError.errorData
-              }
-            });
+            response = serviceErrorToResponse(serviceError);
           }
 
-          Object.keys(response.headers).forEach((key) => {
-            reply.header(key, response.headers[key]);
-          });
-          reply.status(response.status);
           try {
-            await reply.send(response.data);
+            await respond(reply, response);
           } finally {
             await response.afterSend?.({ clientClosed: request.socket.closed });
             requestLogger.info(`${e.method} ${request.url}`, {
@@ -106,3 +103,32 @@ export function registerFastifyRoutes(
     });
   }
 }
+
+/**
+ * Registers a custom not-found handler to ensure 404 error responses have the same schema as other service errors.
+ */
+export function registerFastifyNotFoundHandler(app: fastify.FastifyInstance) {
+  app.setNotFoundHandler(async (request, reply) => {
+    await respond(reply, serviceErrorToResponse(new RouteNotFound(request.originalUrl, request.method)));
+  });
+}
+
+function serviceErrorToResponse(error: ServiceError): router.RouterResponse {
+  return new router.RouterResponse({
+    status: error.errorData.status || 500,
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    data: {
+      error: error.errorData
+    }
+  });
+}
+
+async function respond(reply: FastifyReply, response: router.RouterResponse) {
+  Object.keys(response.headers).forEach((key) => {
+    reply.header(key, response.headers[key]);
+  });
+  reply.status(response.status);
+  await reply.send(response.data);
+}

package/src/storage/BucketStorageBatch.ts

@@ -60,18 +60,29 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
   keepalive(lsn: string): Promise<boolean>;
 
   /**
-   * Set the LSN for a snapshot, before starting replication.
+   * Set the LSN that replication should resume from.
+   *
+   * This can be used for:
+   * 1. Setting the LSN for a snapshot, before starting replication.
+   * 2. Setting the LSN to resume from after a replication restart, without advancing the checkpoint LSN via a commit.
    *
    * Not required if the source database keeps track of this, for example with
    * PostgreSQL logical replication slots.
    */
-  setSnapshotLsn(lsn: string): Promise<void>;
+  setResumeLsn(lsn: string): Promise<void>;
 
   /**
    * Get the last checkpoint LSN, from either commit or keepalive.
    */
   lastCheckpointLsn: string | null;
 
+  /**
+   * LSN to resume from.
+   *
+   * Not relevant for streams where the source keeps track of replication progress, such as Postgres.
+   */
+  resumeFromLsn: string | null;
+
   markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
 
   updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;

package/src/storage/SourceEntity.ts

@@ -10,17 +10,17 @@ export interface ColumnDescriptor {
   typeId?: number;
 }
 
-// TODO: This needs to be consolidated with SourceTable into something new.
 export interface SourceEntityDescriptor {
   /**
-   * The internal id of the data source structure in the database.
-   *
+   * The internal id of the source entity structure in the database.
    * If undefined, the schema and name are used as the identifier.
-   *
    * If specified, this is specifically used to detect renames.
    */
   objectId: number | string | undefined;
   schema: string;
   name: string;
-  replicationColumns: ColumnDescriptor[];
+  /**
+   *  The columns that are used to uniquely identify a record in the source entity.
+   */
+  replicaIdColumns: ColumnDescriptor[];
 }

package/src/storage/SourceTable.ts

@@ -1,6 +1,16 @@
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
-import { ColumnDescriptor } from './SourceEntity.js';
+import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
+
+export interface SourceTableOptions {
+  id: any;
+  connectionTag: string;
+  objectId: number | string | undefined;
+  schema: string;
+  name: string;
+  replicaIdColumns: ColumnDescriptor[];
+  snapshotComplete: boolean;
+}
 
 export interface TableSnapshotStatus {
   totalEstimatedCount: number;
@@ -8,7 +18,7 @@ export interface TableSnapshotStatus {
   lastKey: Uint8Array | null;
 }
 
-export class SourceTable {
+export class SourceTable implements SourceEntityDescriptor {
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
   /**
@@ -45,37 +55,41 @@ export class SourceTable {
    */
   public snapshotStatus: TableSnapshotStatus | undefined = undefined;
 
-  constructor(
-    public readonly id: any,
-    public readonly connectionTag: string,
-    public readonly objectId: number | string | undefined,
-    public readonly schema: string,
-    public readonly table: string,
+  public snapshotComplete: boolean;
 
-    public readonly replicaIdColumns: ColumnDescriptor[],
-    public snapshotComplete: boolean
-  ) {}
+  constructor(public readonly options: SourceTableOptions) {
+    this.snapshotComplete = options.snapshotComplete;
+  }
 
-  get hasReplicaIdentity() {
-    return this.replicaIdColumns.length > 0;
+  get id() {
+    return this.options.id;
   }
 
-  /**
-   * Use for postgres only.
-   *
-   * Usage: db.query({statement: `SELECT $1::regclass`, params: [{type: 'varchar', value: table.qualifiedName}]})
-   */
-  get qualifiedName() {
-    return this.escapedIdentifier;
+  get connectionTag() {
+    return this.options.connectionTag;
+  }
+
+  get objectId() {
+    return this.options.objectId;
+  }
+
+  get schema() {
+    return this.options.schema;
+  }
+  get name() {
+    return this.options.name;
+  }
+
+  get replicaIdColumns() {
+    return this.options.replicaIdColumns;
   }
 
   /**
-   * Use for postgres and logs only.
-   *
-   * Usage: db.query(`SELECT * FROM ${table.escapedIdentifier}`)
+   *  Sanitized name of the entity in the format of "{schema}.{entity name}"
+   *  Suitable for safe use in Postgres queries.
    */
-  get escapedIdentifier() {
-    return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.table)}`;
+  get qualifiedName() {
+    return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.name)}`;
   }
 
   get syncAny() {
@@ -86,15 +100,15 @@ export class SourceTable {
    * In-memory clone of the table status.
    */
   clone() {
-    const copy = new SourceTable(
-      this.id,
-      this.connectionTag,
-      this.objectId,
-      this.schema,
-      this.table,
-      this.replicaIdColumns,
-      this.snapshotComplete
-    );
+    const copy = new SourceTable({
+      id: this.id,
+      connectionTag: this.connectionTag,
+      objectId: this.objectId,
+      schema: this.schema,
+      name: this.name,
+      replicaIdColumns: this.replicaIdColumns,
+      snapshotComplete: this.snapshotComplete
+    });
     copy.syncData = this.syncData;
     copy.syncParameters = this.syncParameters;
     copy.snapshotStatus = this.snapshotStatus;

package/src/storage/SyncRulesBucketStorage.ts

@@ -50,8 +50,6 @@ export interface SyncRulesBucketStorage
    */
   clear(options?: ClearStorageOptions): Promise<void>;
 
-  autoActivate(): Promise<void>;
-
   /**
    * Record a replication error.
    *
@@ -68,11 +66,6 @@ export interface SyncRulesBucketStorage
 
   getCheckpoint(): Promise<ReplicationCheckpoint>;
 
-  /**
-   * Used to resolve "dynamic" parameter queries.
-   */
-  getParameterSets(checkpoint: util.InternalOpId, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
-
   /**
    * Given two checkpoints, return the changes in bucket data and parameters that may have occurred
    * in that period.
@@ -200,6 +193,8 @@ export interface CompactOptions {
    */
   compactBuckets?: string[];
 
+  compactParameterData?: boolean;
+
   /** Minimum of 2 */
   clearBatchLimit?: number;
 
@@ -208,6 +203,11 @@ export interface CompactOptions {
 
   /** Minimum of 1 */
   moveBatchQueryLimit?: number;
+
+  /**
+   * Internal/testing use: Cache size for compacting parameters.
+   */
+  compactParameterCacheLimit?: number;
 }
 
 export interface ClearStorageOptions {
@@ -245,6 +245,13 @@ export interface SyncBucketDataChunk {
 export interface ReplicationCheckpoint {
   readonly checkpoint: util.InternalOpId;
   readonly lsn: string | null;
+
+  /**
+   * Used to resolve "dynamic" parameter queries.
+   *
+   * This gets parameter sets specific to this checkpoint.
+   */
+  getParameterSets(lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
 }
 
 export interface WatchWriteCheckpointOptions {

package/src/sync/BucketChecksumState.ts

@@ -440,7 +440,7 @@ export class BucketParameterState {
     if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
       dynamicBuckets = await querier.queryDynamicBucketDescriptions({
         getParameterSets(lookups) {
-          return storage.getParameterSets(checkpoint.base.checkpoint, lookups);
+          return checkpoint.base.getParameterSets(lookups);
         }
       });
       this.cachedDynamicBuckets = dynamicBuckets;
@@ -501,7 +501,7 @@ export interface CheckpointLine {
 }
 
 // Use a more specific type to simplify testing
-export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums' | 'getParameterSets'>;
+export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
 
 function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
   buckets = buckets.map((b) => {

package/src/sync/util.ts

@@ -3,6 +3,7 @@ import * as timers from 'timers/promises';
 import { SemaphoreInterface } from 'async-mutex';
 import * as util from '../util/util-index.js';
 import { RequestTracker } from './RequestTracker.js';
+import { serialize } from 'bson';
 
 export type TokenStreamOptions = {
   /**
@@ -76,6 +77,27 @@ export async function* tokenStream(
   }
 }
 
+export function syncLineToBson(line: string | Record<string, any>): Buffer {
+  if (typeof line == 'string') {
+    // Should not happen with binary_data: true
+    throw new Error(`Unexpected string data: ${line}`);
+  } else {
+    // On NodeJS, serialize always returns a Buffer
+    return serialize(line) as Buffer;
+  }
+}
+
+export async function* bsonLines(iterator: AsyncIterable<string | null | Record<string, any>>): AsyncGenerator<Buffer> {
+  for await (let line of iterator) {
+    if (line == null) {
+      // Empty value just to flush iterator memory
+      continue;
+    } else {
+      yield syncLineToBson(line);
+    }
+  }
+}
+
 export async function* ndjson(iterator: AsyncIterable<string | null | Record<string, any>>): AsyncGenerator<string> {
   for await (let data of iterator) {
     if (data == null) {
@@ -91,11 +113,18 @@ export async function* ndjson(iterator: AsyncIterable<string | null | Record<str
 }
 
 export async function* transformToBytesTracked(
-  iterator: AsyncIterable<string>,
+  iterator: AsyncIterable<string | Buffer>,
   tracker: RequestTracker
 ): AsyncGenerator<Buffer> {
   for await (let data of iterator) {
-    const encoded = Buffer.from(data, 'utf8');
+    let encoded: Buffer;
+
+    if (typeof data == 'string') {
+      encoded = Buffer.from(data, 'utf8');
+    } else {
+      encoded = data;
+    }
+
     tracker.addDataSynced(encoded.length);
     yield encoded;
   }

package/src/util/config/compound-config-collector.ts

@@ -89,6 +89,7 @@ export class CompoundConfigCollector {
           }
         ])
       );
+      keyStore.supabaseAuthDebug.sharedSecretEnabled = true;
     }
 
     let jwks_uris = baseConfig.client_auth?.jwks_uri ?? [];
@@ -114,6 +115,29 @@ export class CompoundConfigCollector {
     for (let uri of jwks_uris) {
       collectors.add(new auth.CachedKeyCollector(new auth.RemoteJWKSCollector(uri, { lookupOptions: jwksLookup })));
     }
+    const supabaseAuthDetails = auth.getSupabaseJwksUrl(baseConfig.replication?.connections?.[0]);
+    keyStore.supabaseAuthDebug.jwksDetails = supabaseAuthDetails;
+
+    if (baseConfig.client_auth?.supabase) {
+      // Automatic support for Supabase signing keys:
+      // https://supabase.com/docs/guides/auth/signing-keys
+      if (supabaseAuthDetails != null) {
+        const collector = new auth.RemoteJWKSCollector(supabaseAuthDetails.url, {
+          lookupOptions: jwksLookup,
+          // Special case aud and max lifetime for Supabase keys
+          keyOptions: auth.SUPABASE_KEY_OPTIONS
+        });
+        collectors.add(new auth.CachedKeyCollector(collector));
+        keyStore.supabaseAuthDebug.jwksEnabled = true;
+        logger.info(`Configured Supabase Auth with ${supabaseAuthDetails.url}`);
+      } else {
+        logger.warn(
+          'Supabase Auth is enabled, but no Supabase connection string found. Skipping Supabase JWKS URL configuration.'
+        );
+      }
+    } else if (supabaseAuthDetails != null) {
+      logger.warn(`Supabase connection string found, but Supabase Auth is not enabled in the config.`);
+    }
 
     const sync_rules = await this.collectSyncRules(baseConfig, runnerConfig);
 

package/src/util/lsn.ts

@@ -0,0 +1,8 @@
+/**
+ * Return the larger of two LSNs.
+ */
+export function maxLsn(a: string | null | undefined, b: string | null | undefined): string | null {
+  if (a == null) return b ?? null;
+  if (b == null) return a;
+  return a > b ? a : b;
+}

package/src/util/util-index.ts

@@ -1,5 +1,6 @@
 export * from './alerting.js';
 export * from './env.js';
+export * from './lsn.js';
 export * from './memory-tracking.js';
 export * from './Mutex.js';
 export * from './protocol-types.js';