@powersync/service-core 1.14.0 → 1.15.0

package/src/routes/compression.ts

@@ -0,0 +1,75 @@
+import type Negotiator from 'negotiator';
+import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
+import * as zlib from 'node:zlib';
+import { RequestTracker } from '../sync/RequestTracker.js';
+
+/**
+ * Compress a streamed response.
+ *
+ * `@fastify/compress` can do something similar, but does not appear to work as well on streamed responses.
+ * The manual implementation is simple enough, and gives us more control over the low-level details.
+ *
+ * @param negotiator Negotiator from the request, to negotiate response encoding
+ * @param stream plain-text stream
+ * @returns
+ */
+export function maybeCompressResponseStream(
+  negotiator: Negotiator,
+  stream: Readable,
+  tracker: RequestTracker
+): { stream: Readable; encodingHeaders: { 'content-encoding'?: string } } {
+  const encoding = (negotiator as any).encoding(['identity', 'gzip', 'zstd'], { preferred: 'zstd' });
+  const transform = createCompressionTransform(encoding);
+  if (transform == null) {
+    // No matching compression supported - leave stream as-is
+    return {
+      stream,
+      encodingHeaders: {}
+    };
+  } else {
+    tracker.setCompressed(encoding);
+    return {
+      stream: transformStream(stream, transform, tracker),
+      encodingHeaders: { 'content-encoding': encoding }
+    };
+  }
+}
+
+function createCompressionTransform(encoding: string | undefined): Transform | null {
+  if (encoding == 'zstd') {
+    // Available since Node v23.8.0, v22.15.0
+    // This does the actual compression in a background thread pool.
+    return zlib.createZstdCompress({
+      // We need to flush the frame after every new input chunk, to avoid delaying data
+      // in the output stream.
+      flush: zlib.constants.ZSTD_e_flush,
+      params: {
+        // Default compression level is 3. We reduce this slightly to limit CPU overhead
+        [zlib.constants.ZSTD_c_compressionLevel]: 2
+      }
+    });
+  } else if (encoding == 'gzip') {
+    return zlib.createGzip({
+      // We need to flush the frame after every new input chunk, to avoid delaying data
+      // in the output stream.
+      flush: zlib.constants.Z_SYNC_FLUSH
+    });
+  }
+  return null;
+}
+
+function transformStream(source: Readable, transform: Transform, tracker: RequestTracker) {
+  // pipe does not forward error events automatically, resulting in unhandled error
+  // events. This forwards it.
+  const out = new PassThrough();
+  const trackingTransform = new Transform({
+    transform(chunk, _encoding, callback) {
+      tracker.addCompressedDataSent(chunk.length);
+      callback(null, chunk);
+    }
+  });
+  pipeline(source, transform, trackingTransform, out, (err) => {
+    if (err) out.destroy(err);
+  });
+  return out;
+}

package/src/routes/endpoints/socket-route.ts

@@ -11,7 +11,7 @@ import { APIMetric } from '@powersync/service-types';
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
-    handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
+    handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal, connection }) => {
       const { service_context, logger } = context;
       const { routerEngine, metricsEngine, syncContext } = service_context;
 
@@ -58,8 +58,6 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
-      const syncParams = new RequestParameters(context.token_payload!, params.parameters ?? {});
-
       const {
         storageEngine: { activeBucketStorage }
       } = service_context;
@@ -86,16 +84,21 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
 
       metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
       const tracker = new sync.RequestTracker(metricsEngine);
+      if (connection.tracker.encoding) {
+        // Must be set before we start the stream
+        tracker.setCompressed(connection.tracker.encoding);
+      }
       try {
         for await (const data of sync.streamResponse({
           syncContext: syncContext,
           bucketStorage: bucketStorage,
-          syncRules: syncRules,
+          syncRules: {
+            syncRules,
+            version: bucketStorage.group_id
+          },
           params: {
-            ...params,
-            binary_data: true // always true for web sockets
+            ...params
           },
-          syncParams,
           token: context!.token_payload!,
           tokenStreamOptions: {
             // RSocket handles keepalive events by default
@@ -103,7 +106,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
           },
           tracker,
           signal,
-          logger
+          logger,
+          isEncodingAsBson: true
         })) {
           if (signal.aborted) {
             break;
@@ -116,7 +120,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             const serialized = sync.syncLineToBson(data);
             responder.onNext({ data: serialized }, false);
             requestedN--;
-            tracker.addDataSynced(serialized.length);
+            tracker.addPlaintextDataSynced(serialized.length);
           }
 
           if (requestedN <= 0 && !signal.aborted) {
@@ -153,6 +157,17 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         responder.onComplete();
         removeStopHandler();
         disposer();
+        if (connection.tracker.encoding) {
+          // Technically, this may not be unique to this specific stream, since there could be multiple
+          // rsocket streams on the same websocket connection. We don't have a way to track compressed bytes
+          // on individual streams, and we generally expect 1 stream per connection, so this is a reasonable
+          // approximation.
+          // If there are multiple streams, bytes written would be split arbitrarily across them, but the
+          // total should be correct.
+          // For non-compressed cases, this is tracked by the stream itself.
+          const socketBytes = connection.tracker.getBytesWritten();
+          tracker.addCompressedDataSent(socketBytes);
+        }
         logger.info(`Sync stream complete`, {
           ...tracker.getLogMeta(),
           stream_ms: Date.now() - streamStart,

package/src/routes/endpoints/sync-rules.ts

@@ -202,34 +202,7 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
 
     return {
       valid: true,
-      bucket_definitions: rules.bucketDescriptors.map((d) => {
-        let all_parameter_queries = [...d.parameterQueries.values()].flat();
-        let all_data_queries = [...d.dataQueries.values()].flat();
-        return {
-          name: d.name,
-          bucket_parameters: d.bucketParameters,
-          global_parameter_queries: d.globalParameterQueries.map((q) => {
-            return {
-              sql: q.sql
-            };
-          }),
-          parameter_queries: all_parameter_queries.map((q) => {
-            return {
-              sql: q.sql,
-              table: q.sourceTable,
-              input_parameters: q.inputParameters
-            };
-          }),
-
-          data_queries: all_data_queries.map((q) => {
-            return {
-              sql: q.sql,
-              table: q.sourceTable,
-              columns: q.columnOutputNames()
-            };
-          })
-        };
-      }),
+      bucket_definitions: rules.bucketSources.map((source) => source.debugRepresentation()),
       source_tables: resolved_tables,
       data_tables: rules.debugGetOutputTables()
     };

package/src/routes/endpoints/sync-stream.ts

@@ -1,7 +1,6 @@
-import { ErrorCode, errors, logger, router, schema } from '@powersync/lib-services-framework';
-import { RequestParameters } from '@powersync/service-sync-rules';
-import { Readable } from 'stream';
+import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
 import Negotiator from 'negotiator';
+import { Readable } from 'stream';
 
 import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
@@ -10,6 +9,7 @@ import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
 import { APIMetric } from '@powersync/service-types';
+import { maybeCompressResponseStream } from '../compression.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -31,10 +31,11 @@ export const syncStreamed = routeDefinition({
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
     const clientId = payload.params.client_id;
     const streamStart = Date.now();
+    const negotiator = new Negotiator(payload.request);
     // This falls back to JSON unless there's preference for the bson-stream in the Accept header.
-    const useBson =
-      payload.request.headers.accept &&
-      new Negotiator(payload.request).mediaType(supportedContentTypes) == concatenatedBsonContentType;
+    const useBson = payload.request.headers.accept
+      ? negotiator.mediaType(supportedContentTypes) == concatenatedBsonContentType
+      : false;
 
     logger.defaultMeta = {
       ...logger.defaultMeta,
@@ -52,9 +53,6 @@ export const syncStreamed = routeDefinition({
       });
     }
 
-    const params: util.StreamingSyncRequest = payload.params;
-    const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
-
     const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage();
 
     if (bucketStorage == null) {
@@ -74,20 +72,24 @@ export const syncStreamed = routeDefinition({
       const syncLines = sync.streamResponse({
         syncContext: syncContext,
         bucketStorage,
-        syncRules: syncRules,
-        params,
-        syncParams,
+        syncRules: {
+          syncRules,
+          version: bucketStorage.group_id
+        },
+        params: payload.params,
         token: payload.context.token_payload!,
         tracker,
         signal: controller.signal,
-        logger
+        logger,
+        isEncodingAsBson: useBson
       });
 
       const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);
-      const stream = Readable.from(sync.transformToBytesTracked(byteContents, tracker), {
+      const plainStream = Readable.from(sync.transformToBytesTracked(byteContents, tracker), {
         objectMode: false,
         highWaterMark: 16 * 1024
       });
+      const { stream, encodingHeaders } = maybeCompressResponseStream(negotiator, plainStream, tracker);
 
       // Best effort guess on why the stream was closed.
       // We use the `??=` operator everywhere, so that we catch the first relevant
@@ -122,7 +124,8 @@ export const syncStreamed = routeDefinition({
       return new router.RouterResponse({
         status: 200,
         headers: {
-          'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType
+          'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType,
+          ...encodingHeaders
         },
         data: stream,
         afterSend: async (details) => {

package/src/storage/BucketStorage.ts

@@ -39,8 +39,8 @@ export enum SyncRuleState {
 export const DEFAULT_DOCUMENT_BATCH_LIMIT = 1000;
 export const DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES = 1 * 1024 * 1024;
 
-export function mergeToast(record: ToastableSqliteRow, persisted: ToastableSqliteRow): ToastableSqliteRow {
-  const newRecord: ToastableSqliteRow = {};
+export function mergeToast<V>(record: ToastableSqliteRow<V>, persisted: ToastableSqliteRow<V>): ToastableSqliteRow<V> {
+  const newRecord: ToastableSqliteRow<V> = {};
   for (let key in record) {
     if (typeof record[key] == 'undefined') {
       newRecord[key] = persisted[key];

package/src/storage/BucketStorageBatch.ts

@@ -1,5 +1,11 @@
 import { ObserverClient } from '@powersync/lib-services-framework';
-import { EvaluatedParameters, EvaluatedRow, SqliteRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
+import {
+  EvaluatedParameters,
+  EvaluatedRow,
+  SqliteInputRow,
+  SqliteRow,
+  ToastableSqliteRow
+} from '@powersync/service-sync-rules';
 import { BSON } from 'bson';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
@@ -132,7 +138,7 @@ export interface SaveInsert {
   sourceTable: SourceTable;
   before?: undefined;
   beforeReplicaId?: undefined;
-  after: SqliteRow;
+  after: SqliteInputRow;
   afterReplicaId: ReplicaId;
 }
 
@@ -143,7 +149,7 @@ export interface SaveUpdate {
   /**
    * This is only present when the id has changed, and will only contain replica identity columns.
    */
-  before?: SqliteRow;
+  before?: SqliteInputRow;
   beforeReplicaId?: ReplicaId;
 
   /**
@@ -158,7 +164,7 @@ export interface SaveUpdate {
 export interface SaveDelete {
   tag: SaveOperationTag.DELETE;
   sourceTable: SourceTable;
-  before?: SqliteRow;
+  before?: SqliteInputRow;
   beforeReplicaId: ReplicaId;
   after?: undefined;
   afterReplicaId?: undefined;

package/src/storage/ChecksumCache.ts

@@ -1,40 +1,21 @@
 import { OrderedSet } from '@js-sdsl/ordered-set';
 import { LRUCache } from 'lru-cache/min';
 import { BucketChecksum } from '../util/protocol-types.js';
-import { addBucketChecksums, ChecksumMap, InternalOpId } from '../util/utils.js';
+import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
 
 interface ChecksumFetchContext {
   fetch(bucket: string): Promise<BucketChecksum>;
   checkpoint: InternalOpId;
 }
 
-export interface PartialChecksum {
-  bucket: string;
-  /**
-   * 32-bit unsigned hash.
-   */
-  partialChecksum: number;
-
-  /**
-   * Count of operations - informational only.
-   */
-  partialCount: number;
-
-  /**
-   * True if the queried operations contains (starts with) a CLEAR
-   * operation, indicating that the partial checksum is the full
-   * checksum, and must not be added to a previously-cached checksum.
-   */
-  isFullChecksum: boolean;
-}
-
 export interface FetchPartialBucketChecksum {
   bucket: string;
   start?: InternalOpId;
   end: InternalOpId;
 }
 
-export type PartialChecksumMap = Map<string, PartialChecksum>;
+export type PartialOrFullChecksum = PartialChecksum | BucketChecksum;
+export type PartialChecksumMap = Map<string, PartialOrFullChecksum>;
 
 export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<PartialChecksumMap>;
 
@@ -127,6 +108,11 @@ export class ChecksumCache {
     });
   }
 
+  clear() {
+    this.cache.clear();
+    this.bucketCheckpoints.clear();
+  }
+
   async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]> {
     const checksums = await this.getChecksumMap(checkpoint, buckets);
     // Return results in the same order as the request

package/src/storage/ReplicationEventPayload.ts

@@ -4,8 +4,8 @@ import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
 
 export type EventData = {
   op: SaveOp;
-  before?: sync_rules.SqliteRow;
-  after?: sync_rules.SqliteRow;
+  before?: sync_rules.SqliteInputRow;
+  after?: sync_rules.SqliteInputRow;
 };
 
 export type ReplicationEventPayload = {

package/src/storage/SyncRulesBucketStorage.ts

@@ -62,6 +62,11 @@ export interface SyncRulesBucketStorage
 
   compact(options?: CompactOptions): Promise<void>;
 
+  /**
+   * Lightweight "compact" process to populate the checksum cache, if any.
+   */
+  populatePersistentChecksumCache(options?: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void>;
+
   // ## Read operations
 
   getCheckpoint(): Promise<ReplicationCheckpoint>;
@@ -108,6 +113,11 @@ export interface SyncRulesBucketStorage
    * Returns zero checksums for any buckets not found.
    */
   getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
+
+  /**
+   * Clear checksum cache. Primarily intended for tests.
+   */
+  clearChecksumCache(): void;
 }
 
 export interface SyncRulesBucketStorageListener {
@@ -208,6 +218,8 @@ export interface CompactOptions {
    * Internal/testing use: Cache size for compacting parameters.
    */
   compactParameterCacheLimit?: number;
+
+  signal?: AbortSignal;
 }
 
 export interface ClearStorageOptions {