@powersync/service-core 1.13.4 → 1.15.0

package/src/storage/SourceTable.ts

@@ -1,6 +1,16 @@
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
-import { ColumnDescriptor } from './SourceEntity.js';
+import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
+
+export interface SourceTableOptions {
+  id: any;
+  connectionTag: string;
+  objectId: number | string | undefined;
+  schema: string;
+  name: string;
+  replicaIdColumns: ColumnDescriptor[];
+  snapshotComplete: boolean;
+}
 
 export interface TableSnapshotStatus {
   totalEstimatedCount: number;
@@ -8,7 +18,7 @@ export interface TableSnapshotStatus {
   lastKey: Uint8Array | null;
 }
 
-export class SourceTable {
+export class SourceTable implements SourceEntityDescriptor {
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
   /**
@@ -45,37 +55,41 @@ export class SourceTable {
    */
   public snapshotStatus: TableSnapshotStatus | undefined = undefined;
 
-  constructor(
-    public readonly id: any,
-    public readonly connectionTag: string,
-    public readonly objectId: number | string | undefined,
-    public readonly schema: string,
-    public readonly table: string,
+  public snapshotComplete: boolean;
 
-    public readonly replicaIdColumns: ColumnDescriptor[],
-    public snapshotComplete: boolean
-  ) {}
+  constructor(public readonly options: SourceTableOptions) {
+    this.snapshotComplete = options.snapshotComplete;
+  }
 
-  get hasReplicaIdentity() {
-    return this.replicaIdColumns.length > 0;
+  get id() {
+    return this.options.id;
   }
 
-  /**
-   * Use for postgres only.
-   *
-   * Usage: db.query({statement: `SELECT $1::regclass`, params: [{type: 'varchar', value: table.qualifiedName}]})
-   */
-  get qualifiedName() {
-    return this.escapedIdentifier;
+  get connectionTag() {
+    return this.options.connectionTag;
+  }
+
+  get objectId() {
+    return this.options.objectId;
+  }
+
+  get schema() {
+    return this.options.schema;
+  }
+  get name() {
+    return this.options.name;
+  }
+
+  get replicaIdColumns() {
+    return this.options.replicaIdColumns;
   }
 
   /**
-   * Use for postgres and logs only.
-   *
-   * Usage: db.query(`SELECT * FROM ${table.escapedIdentifier}`)
+   *  Sanitized name of the entity in the format of "{schema}.{entity name}"
+   *  Suitable for safe use in Postgres queries.
    */
-  get escapedIdentifier() {
-    return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.table)}`;
+  get qualifiedName() {
+    return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.name)}`;
   }
 
   get syncAny() {
@@ -86,15 +100,15 @@ export class SourceTable {
    * In-memory clone of the table status.
    */
   clone() {
-    const copy = new SourceTable(
-      this.id,
-      this.connectionTag,
-      this.objectId,
-      this.schema,
-      this.table,
-      this.replicaIdColumns,
-      this.snapshotComplete
-    );
+    const copy = new SourceTable({
+      id: this.id,
+      connectionTag: this.connectionTag,
+      objectId: this.objectId,
+      schema: this.schema,
+      name: this.name,
+      replicaIdColumns: this.replicaIdColumns,
+      snapshotComplete: this.snapshotComplete
+    });
     copy.syncData = this.syncData;
     copy.syncParameters = this.syncParameters;
     copy.snapshotStatus = this.snapshotStatus;

package/src/storage/SyncRulesBucketStorage.ts

@@ -50,8 +50,6 @@ export interface SyncRulesBucketStorage
    */
   clear(options?: ClearStorageOptions): Promise<void>;
 
-  autoActivate(): Promise<void>;
-
   /**
    * Record a replication error.
    *
@@ -64,15 +62,15 @@ export interface SyncRulesBucketStorage
 
   compact(options?: CompactOptions): Promise<void>;
 
+  /**
+   * Lightweight "compact" process to populate the checksum cache, if any.
+   */
+  populatePersistentChecksumCache(options?: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void>;
+
   // ## Read operations
 
   getCheckpoint(): Promise<ReplicationCheckpoint>;
 
-  /**
-   * Used to resolve "dynamic" parameter queries.
-   */
-  getParameterSets(checkpoint: util.InternalOpId, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
-
   /**
    * Given two checkpoints, return the changes in bucket data and parameters that may have occurred
    * in that period.
@@ -115,6 +113,11 @@ export interface SyncRulesBucketStorage
    * Returns zero checksums for any buckets not found.
    */
   getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
+
+  /**
+   * Clear checksum cache. Primarily intended for tests.
+   */
+  clearChecksumCache(): void;
 }
 
 export interface SyncRulesBucketStorageListener {
@@ -200,6 +203,8 @@ export interface CompactOptions {
    */
   compactBuckets?: string[];
 
+  compactParameterData?: boolean;
+
   /** Minimum of 2 */
   clearBatchLimit?: number;
 
@@ -208,6 +213,13 @@ export interface CompactOptions {
 
   /** Minimum of 1 */
   moveBatchQueryLimit?: number;
+
+  /**
+   * Internal/testing use: Cache size for compacting parameters.
+   */
+  compactParameterCacheLimit?: number;
+
+  signal?: AbortSignal;
 }
 
 export interface ClearStorageOptions {
@@ -245,6 +257,13 @@ export interface SyncBucketDataChunk {
 export interface ReplicationCheckpoint {
   readonly checkpoint: util.InternalOpId;
   readonly lsn: string | null;
+
+  /**
+   * Used to resolve "dynamic" parameter queries.
+   *
+   * This gets parameter sets specific to this checkpoint.
+   */
+  getParameterSets(lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
 }
 
 export interface WatchWriteCheckpointOptions {

package/src/sync/BucketChecksumState.ts

@@ -1,4 +1,13 @@
-import { BucketDescription, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
+import {
+  BucketDescription,
+  BucketPriority,
+  BucketSource,
+  RequestedStream,
+  RequestJwtPayload,
+  RequestParameters,
+  ResolvedBucket,
+  SqlSyncRules
+} from '@powersync/service-sync-rules';
 
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
@@ -11,17 +20,22 @@ import {
   logger as defaultLogger
 } from '@powersync/lib-services-framework';
 import { JSONBig } from '@powersync/service-jsonbig';
-import { BucketParameterQuerier } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
+import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
 import { SyncContext } from './SyncContext.js';
 import { getIntersection, hasIntersection } from './util.js';
 
+export interface VersionedSyncRules {
+  syncRules: SqlSyncRules;
+  version: number;
+}
+
 export interface BucketChecksumStateOptions {
   syncContext: SyncContext;
   bucketStorage: BucketChecksumStateStorage;
-  syncRules: SqlSyncRules;
-  syncParams: RequestParameters;
+  syncRules: VersionedSyncRules;
+  tokenPayload: RequestJwtPayload;
+  syncRequest: util.StreamingSyncRequest;
   logger?: Logger;
-  initialBucketPositions?: { name: string; after: util.InternalOpId }[];
 }
 
 type BucketSyncState = {
@@ -50,6 +64,17 @@ export class BucketChecksumState {
    */
   private lastChecksums: util.ChecksumMap | null = null;
   private lastWriteCheckpoint: bigint | null = null;
+  /**
+   * Once we've sent the first full checkpoint line including all {@link util.Checkpoint.streams} that the user is
+   * subscribed to, we keep an index of the stream names to their index in that array.
+   *
+   * This is used to compress the representation of buckets in `checkpoint` and `checkpoint_diff` lines: For buckets
+   * that are part of sync rules or default streams, we need to include the name of the defining sync rule or definition
+   * yielding that bucket (so that clients can track progress for default streams).
+   * But instead of sending the name for each bucket, we use the fact that it's part of the streams array and only send
+   * their index, reducing the size of those messages.
+   */
+  private streamNameToIndex: Map<string, number> | null = null;
 
   private readonly parameterState: BucketParameterState;
 
@@ -69,13 +94,14 @@ export class BucketChecksumState {
       options.syncContext,
       options.bucketStorage,
       options.syncRules,
-      options.syncParams,
+      options.tokenPayload,
+      options.syncRequest,
       this.logger
     );
     this.bucketDataPositions = new Map();
 
-    for (let { name, after: start } of options.initialBucketPositions ?? []) {
-      this.bucketDataPositions.set(name, { start_op_id: start });
+    for (let { name, after: start } of options.syncRequest.buckets ?? []) {
+      this.bucketDataPositions.set(name, { start_op_id: BigInt(start) });
     }
   }
 
@@ -158,6 +184,7 @@ export class BucketChecksumState {
       // TODO: If updatedBuckets is present, we can use that to more efficiently calculate a diff,
       // and avoid any unnecessary loops through the entire list of buckets.
       const diff = util.checksumsDiff(this.lastChecksums, checksumMap);
+      const streamNameToIndex = this.streamNameToIndex!;
 
       if (
         this.lastWriteCheckpoint == writeCheckpoint &&
@@ -182,12 +209,12 @@ export class BucketChecksumState {
 
       const updatedBucketDescriptions = diff.updatedBuckets.map((e) => ({
         ...e,
-        priority: bucketDescriptionMap.get(e.bucket)!.priority
+        ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex)
       }));
       bucketsToFetch = [...generateBucketsToFetch].map((b) => {
         return {
-          bucket: b,
-          priority: bucketDescriptionMap.get(b)!.priority
+          priority: bucketDescriptionMap.get(b)!.priority,
+          bucket: b
         };
       });
 
@@ -220,15 +247,37 @@ export class BucketChecksumState {
         message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
         this.logger.info(message, { checkpoint: base.checkpoint, user_id: user_id, buckets: allBuckets.length });
       };
-      bucketsToFetch = allBuckets;
+      bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
+
+      const subscriptions: util.StreamDescription[] = [];
+      const streamNameToIndex = new Map<string, number>();
+      this.streamNameToIndex = streamNameToIndex;
+
+      for (const source of this.parameterState.syncRules.syncRules.bucketSources) {
+        if (this.parameterState.isSubscribedToStream(source)) {
+          streamNameToIndex.set(source.name, subscriptions.length);
+
+          subscriptions.push({
+            name: source.name,
+            is_default: source.subscribedToByDefault,
+            errors:
+              this.parameterState.streamErrors[source.name]?.map((e) => ({
+                subscription: e.subscription?.opaque_id ?? 'default',
+                message: e.message
+              })) ?? []
+          });
+        }
+      }
+
       checkpointLine = {
         checkpoint: {
           last_op_id: util.internalToExternalOpId(base.checkpoint),
           write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
           buckets: [...checksumMap.values()].map((e) => ({
             ...e,
-            priority: bucketDescriptionMap.get(e.bucket)!.priority
-          }))
+            ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex)
+          })),
+          streams: subscriptions
         }
       } satisfies util.StreamingSyncCheckpoint;
     }
@@ -319,7 +368,7 @@ export interface CheckpointUpdate {
   /**
    * All buckets forming part of the checkpoint.
    */
-  buckets: BucketDescription[];
+  buckets: ResolvedBucket[];
 
   /**
    * If present, a set of buckets that have been updated since the last checkpoint.
@@ -332,12 +381,22 @@ export interface CheckpointUpdate {
 export class BucketParameterState {
   private readonly context: SyncContext;
   public readonly bucketStorage: BucketChecksumStateStorage;
-  public readonly syncRules: SqlSyncRules;
+  public readonly syncRules: VersionedSyncRules;
   public readonly syncParams: RequestParameters;
   private readonly querier: BucketParameterQuerier;
-  private readonly staticBuckets: Map<string, BucketDescription>;
+  /**
+   * Static buckets. This map is guaranteed not to change during a request, since resolving static buckets can only
+   * take request parameters into account,
+   */
+  private readonly staticBuckets: Map<string, ResolvedBucket>;
+  private readonly includeDefaultStreams: boolean;
+  // Indexed by the client-side id
+  private readonly explicitStreamSubscriptions: util.RequestedStreamSubscription[];
+  // Indexed by descriptor name.
+  readonly streamErrors: Record<string, QuerierError[]>;
+  private readonly subscribedStreamNames: Set<string>;
   private readonly logger: Logger;
-  private cachedDynamicBuckets: BucketDescription[] | null = null;
+  private cachedDynamicBuckets: ResolvedBucket[] | null = null;
   private cachedDynamicBucketSet: Set<string> | null = null;
 
   private readonly lookups: Set<string>;
@@ -345,19 +404,94 @@ export class BucketParameterState {
   constructor(
     context: SyncContext,
     bucketStorage: BucketChecksumStateStorage,
-    syncRules: SqlSyncRules,
-    syncParams: RequestParameters,
+    syncRules: VersionedSyncRules,
+    tokenPayload: RequestJwtPayload,
+    request: util.StreamingSyncRequest,
     logger: Logger
   ) {
     this.context = context;
     this.bucketStorage = bucketStorage;
     this.syncRules = syncRules;
-    this.syncParams = syncParams;
+    this.syncParams = new RequestParameters(tokenPayload, request.parameters ?? {});
     this.logger = logger;
 
-    this.querier = syncRules.getBucketParameterQuerier(this.syncParams);
-    this.staticBuckets = new Map<string, BucketDescription>(this.querier.staticBuckets.map((b) => [b.bucket, b]));
+    const streamsByName: Record<string, RequestedStream[]> = {};
+    const subscriptions = request.streams;
+    const explicitStreamSubscriptions: util.RequestedStreamSubscription[] = subscriptions?.subscriptions ?? [];
+    if (subscriptions) {
+      for (let i = 0; i < explicitStreamSubscriptions.length; i++) {
+        const subscription = explicitStreamSubscriptions[i];
+
+        const syncRuleStream: RequestedStream = {
+          parameters: subscription.parameters ?? {},
+          opaque_id: i
+        };
+        if (Object.hasOwn(streamsByName, subscription.stream)) {
+          streamsByName[subscription.stream].push(syncRuleStream);
+        } else {
+          streamsByName[subscription.stream] = [syncRuleStream];
+        }
+      }
+    }
+    this.includeDefaultStreams = subscriptions?.include_defaults ?? true;
+    this.explicitStreamSubscriptions = explicitStreamSubscriptions;
+
+    const { querier, errors } = syncRules.syncRules.getBucketParameterQuerier({
+      globalParameters: this.syncParams,
+      hasDefaultStreams: this.includeDefaultStreams,
+      streams: streamsByName,
+      bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${syncRules.version}`)
+    });
+    this.querier = querier;
+    this.streamErrors = Object.groupBy(errors, (e) => e.descriptor) as Record<string, QuerierError[]>;
+
+    this.staticBuckets = new Map<string, ResolvedBucket>(
+      mergeBuckets(this.querier.staticBuckets).map((b) => [b.bucket, b])
+    );
     this.lookups = new Set<string>(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values)));
+    this.subscribedStreamNames = new Set(Object.keys(streamsByName));
+  }
+
+  /**
+   * Translates an internal sync-rules {@link ResolvedBucket} instance to the public
+   * {@link util.ClientBucketDescription}.
+   *
+   * @param lookupIndex A map from stream names to their index in {@link util.Checkpoint.streams}. These are used to
+   * reference default buckets by their stream index instead of duplicating the name on wire.
+   */
+  translateResolvedBucket(description: ResolvedBucket, lookupIndex: Map<string, number>): util.ClientBucketDescription {
+    // If the client is overriding the priority of any stream that yields this bucket, sync the bucket with that
+    // priority.
+    let priorityOverride: BucketPriority | null = null;
+    for (const reason of description.inclusion_reasons) {
+      if (reason != 'default') {
+        const requestedPriority = this.explicitStreamSubscriptions[reason.subscription]?.override_priority;
+        if (requestedPriority != null) {
+          if (priorityOverride == null) {
+            priorityOverride = requestedPriority as BucketPriority;
+          } else {
+            priorityOverride = Math.min(requestedPriority, priorityOverride) as BucketPriority;
+          }
+        }
+      }
+    }
+
+    return {
+      bucket: description.bucket,
+      priority: priorityOverride ?? description.priority,
+      subscriptions: description.inclusion_reasons.map((reason) => {
+        if (reason == 'default') {
+          const stream = description.definition;
+          return { default: lookupIndex.get(stream)! };
+        } else {
+          return { sub: reason.subscription };
+        }
+      })
+    };
+  }
+
+  isSubscribedToStream(desc: BucketSource): boolean {
+    return (desc.subscribedToByDefault && this.includeDefaultStreams) || this.subscribedStreamNames.has(desc.name);
   }
 
   async getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
@@ -391,19 +525,19 @@ export class BucketParameterState {
    * For static buckets, we can keep track of which buckets have been updated.
    */
   private async getCheckpointUpdateStatic(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
-    const querier = this.querier;
+    const staticBuckets = [...this.staticBuckets.values()];
     const update = checkpoint.update;
 
     if (update.invalidateDataBuckets) {
       return {
-        buckets: querier.staticBuckets,
+        buckets: staticBuckets,
         updatedBuckets: INVALIDATE_ALL_BUCKETS
       };
     }
 
     const updatedBuckets = new Set<string>(getIntersection(this.staticBuckets, update.updatedDataBuckets));
     return {
-      buckets: querier.staticBuckets,
+      buckets: staticBuckets,
       updatedBuckets
     };
   }
@@ -414,7 +548,7 @@ export class BucketParameterState {
   private async getCheckpointUpdateDynamic(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
     const querier = this.querier;
     const storage = this.bucketStorage;
-    const staticBuckets = querier.staticBuckets;
+    const staticBuckets = this.staticBuckets.values();
     const update = checkpoint.update;
 
     let hasParameterChange = false;
@@ -436,11 +570,11 @@ export class BucketParameterState {
       }
     }
 
-    let dynamicBuckets: BucketDescription[];
+    let dynamicBuckets: ResolvedBucket[];
     if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
       dynamicBuckets = await querier.queryDynamicBucketDescriptions({
         getParameterSets(lookups) {
-          return storage.getParameterSets(checkpoint.base.checkpoint, lookups);
+          return checkpoint.base.getParameterSets(lookups);
         }
       });
       this.cachedDynamicBuckets = dynamicBuckets;
@@ -458,7 +592,7 @@ export class BucketParameterState {
         }
       }
     }
-    const allBuckets = [...staticBuckets, ...dynamicBuckets];
+    const allBuckets = [...staticBuckets, ...mergeBuckets(dynamicBuckets)];
 
     if (invalidateDataBuckets) {
       return {
@@ -501,7 +635,7 @@ export interface CheckpointLine {
 }
 
 // Use a more specific type to simplify testing
-export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums' | 'getParameterSets'>;
+export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
 
 function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
   buckets = buckets.map((b) => {
@@ -517,3 +651,32 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number)
   const limited = buckets.slice(0, limit);
   return `${JSON.stringify(limited)}...`;
 }
+
+/**
+ * Resolves duplicate buckets in the given array, merging the inclusion reasons for duplicate.
+ *
+ * It's possible for duplicates to occur when a stream has multiple subscriptions, consider e.g.
+ *
+ * ```
+ * sync_streams:
+ *  assets_by_category:
+ *    query: select * from assets where category in (request.parameters() -> 'categories')
+ * ```
+ *
+ * Here, a client might subscribe once with `{"categories": [1]}` and once with `{"categories": [1, 2]}`. Since each
+ * subscription is evaluated independently, this would lead to three buckets, with a duplicate `assets_by_category[1]`
+ * bucket.
+ */
+function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] {
+  const byBucketId: Record<string, ResolvedBucket> = {};
+
+  for (const bucket of buckets) {
+    if (Object.hasOwn(byBucketId, bucket.bucket)) {
+      byBucketId[bucket.bucket].inclusion_reasons.push(...bucket.inclusion_reasons);
+    } else {
+      byBucketId[bucket.bucket] = structuredClone(bucket);
+    }
+  }
+
+  return Object.values(byBucketId);
+}

package/src/sync/RequestTracker.ts

@@ -2,6 +2,7 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js';
 
 import { APIMetric } from '@powersync/service-types';
 import { SyncBucketData } from '../util/protocol-types.js';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 /**
  * Record sync stats per request stream.
@@ -9,9 +10,12 @@ import { SyncBucketData } from '../util/protocol-types.js';
 export class RequestTracker {
   operationsSynced = 0;
   dataSyncedBytes = 0;
+  dataSentBytes = 0;
   operationCounts: OperationCounts = { put: 0, remove: 0, move: 0, clear: 0 };
   largeBuckets: Record<string, number> = {};
 
+  private encoding: string | undefined = undefined;
+
   constructor(private metrics: MetricsEngine) {
     this.metrics = metrics;
   }
@@ -29,18 +33,39 @@ export class RequestTracker {
     this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED).add(operations.total);
   }
 
-  addDataSynced(bytes: number) {
+  setCompressed(encoding: string) {
+    this.encoding = encoding;
+  }
+
+  addPlaintextDataSynced(bytes: number) {
     this.dataSyncedBytes += bytes;
 
     this.metrics.getCounter(APIMetric.DATA_SYNCED_BYTES).add(bytes);
+
+    if (this.encoding == null) {
+      // This avoids having to create a separate stream just to track this
+      this.dataSentBytes += bytes;
+
+      this.metrics.getCounter(APIMetric.DATA_SENT_BYTES).add(bytes);
+    }
+  }
+
+  addCompressedDataSent(bytes: number) {
+    if (this.encoding == null) {
+      throw new ServiceAssertionError('No compression encoding set');
+    }
+    this.dataSentBytes += bytes;
+    this.metrics.getCounter(APIMetric.DATA_SENT_BYTES).add(bytes);
   }
 
   getLogMeta() {
     return {
       operations_synced: this.operationsSynced,
       data_synced_bytes: this.dataSyncedBytes,
+      data_sent_bytes: this.dataSentBytes,
       operation_counts: this.operationCounts,
-      large_buckets: this.largeBuckets
+      large_buckets: this.largeBuckets,
+      encoding: this.encoding
     };
   }
 }