@powersync/service-core 1.19.2 → 1.20.1

package/src/sync/BucketChecksumState.ts

@@ -1,11 +1,14 @@
 import {
   BucketDescription,
+  BucketParameterQuerier,
   BucketPriority,
   BucketSource,
   HydratedSyncRules,
+  QuerierError,
   RequestedStream,
   RequestParameters,
-  ResolvedBucket
+  ResolvedBucket,
+  mergeBuckets
 } from '@powersync/service-sync-rules';
 
 import * as storage from '../storage/storage-index.js';
@@ -18,7 +21,6 @@ import {
   ServiceAssertionError,
   ServiceError
 } from '@powersync/lib-services-framework';
-import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
 import { JwtPayload } from '../auth/JwtPayload.js';
 import { SyncContext } from './SyncContext.js';
 import { getIntersection, hasIntersection } from './util.js';
@@ -117,7 +119,7 @@ export class BucketChecksumState {
     const storage = this.bucketStorage;
 
     const update = await this.parameterState.getCheckpointUpdate(next);
-    const { buckets: allBuckets, updatedBuckets } = update;
+    const { buckets: allBuckets, updatedBuckets, parameterQueryResultsByDefinition } = update;
 
     /** Set of all buckets in this checkpoint. */
     const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
@@ -136,20 +138,20 @@ export class BucketChecksumState {
       }
 
       // Re-check updated buckets only
-      let checksumLookups: string[] = [];
+      let checksumLookups: storage.BucketChecksumRequest[] = [];
 
       let newChecksums = new Map<string, util.BucketChecksum>();
-      for (let bucket of bucketDescriptionMap.keys()) {
-        if (!updatedBuckets.has(bucket)) {
-          const existing = this.lastChecksums.get(bucket);
+      for (let desc of bucketDescriptionMap.values()) {
+        if (!updatedBuckets.has(desc.bucket)) {
+          const existing = this.lastChecksums.get(desc.bucket);
           if (existing == null) {
             // If this happens, it means updatedBuckets did not correctly include all new buckets
-            throw new ServiceAssertionError(`Existing checksum not found for bucket ${bucket}`);
+            throw new ServiceAssertionError(`Existing checksum not found for bucket ${desc.bucket}`);
           }
           // Bucket is not specifically updated, and we have a previous checksum
-          newChecksums.set(bucket, existing);
+          newChecksums.set(desc.bucket, existing);
         } else {
-          checksumLookups.push(bucket);
+          checksumLookups.push({ bucket: desc.bucket, source: desc.source });
         }
       }
 
@@ -162,12 +164,12 @@ export class BucketChecksumState {
       checksumMap = newChecksums;
     } else {
       // Re-check all buckets
-      const bucketList = [...bucketDescriptionMap.keys()];
+      const bucketList = [...bucketDescriptionMap.values()].map((b) => ({ bucket: b.bucket, source: b.source }));
       checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
     }
 
     // Subset of buckets for which there may be new data in this batch.
-    let bucketsToFetch: BucketDescription[];
+    let bucketsToFetch: ResolvedBucket[];
 
     let checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
 
@@ -206,25 +208,31 @@ export class BucketChecksumState {
         ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex)
       }));
       bucketsToFetch = [...generateBucketsToFetch].map((b) => {
-        return {
-          priority: bucketDescriptionMap.get(b)!.priority,
-          bucket: b
-        };
+        return bucketDescriptionMap.get(b)!;
       });
 
       deferredLog = () => {
+        const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
         let message = `Updated checkpoint: ${base.checkpoint} | `;
         message += `write: ${writeCheckpoint} | `;
         message += `buckets: ${allBuckets.length} | `;
+        if (totalParamResults !== undefined) {
+          message += `param_results: ${totalParamResults} | `;
+        }
         message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
         message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
-        this.logger.info(message, {
-          checkpoint: base.checkpoint,
-          user_id: userIdForLogs,
-          buckets: allBuckets.length,
-          updated: diff.updatedBuckets.length,
-          removed: diff.removedBuckets.length
-        });
+        logCheckpoint(
+          this.logger,
+          message,
+          {
+            checkpoint: base.checkpoint,
+            user_id: userIdForLogs,
+            buckets: allBuckets.length,
+            updated: diff.updatedBuckets.length,
+            removed: diff.removedBuckets.length
+          },
+          totalParamResults
+        );
       };
 
       checkpointLine = {
@@ -237,11 +245,25 @@ export class BucketChecksumState {
       } satisfies util.StreamingSyncCheckpointDiff;
     } else {
       deferredLog = () => {
+        const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
         let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
-        message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
-        this.logger.info(message, { checkpoint: base.checkpoint, user_id: userIdForLogs, buckets: allBuckets.length });
+        message += `buckets: ${allBuckets.length}`;
+        if (totalParamResults !== undefined) {
+          message += ` | param_results: ${totalParamResults}`;
+        }
+        message += ` ${limitedBuckets(allBuckets, 20)}`;
+        logCheckpoint(
+          this.logger,
+          message,
+          {
+            checkpoint: base.checkpoint,
+            user_id: userIdForLogs,
+            buckets: allBuckets.length
+          },
+          totalParamResults
+        );
       };
-      bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
+      bucketsToFetch = allBuckets;
 
       const subscriptions: util.StreamDescription[] = [];
       const streamNameToIndex = new Map<string, number>();
@@ -318,17 +340,17 @@ export class BucketChecksumState {
         deferredLog();
       },
 
-      getFilteredBucketPositions: (buckets?: BucketDescription[]): Map<string, util.InternalOpId> => {
+      getFilteredBucketPositions: (buckets?: ResolvedBucket[]): storage.BucketDataRequest[] => {
         if (!hasAdvanced) {
           throw new ServiceAssertionError('Call line.advance() before getFilteredBucketPositions()');
         }
         buckets ??= bucketsToFetch;
-        const filtered = new Map<string, util.InternalOpId>();
+        const filtered: storage.BucketDataRequest[] = [];
 
         for (let bucket of buckets) {
           const state = this.bucketDataPositions.get(bucket.bucket);
           if (state) {
-            filtered.set(bucket.bucket, state.start_op_id);
+            filtered.push({ bucket: bucket.bucket, start: state.start_op_id, source: bucket.source });
           }
         }
         return filtered;
@@ -370,6 +392,12 @@ export interface CheckpointUpdate {
    * If null, assume that any bucket in `buckets` may have been updated.
    */
   updatedBuckets: Set<string> | typeof INVALIDATE_ALL_BUCKETS;
+
+  /**
+   * Number of parameter query results per sync stream definition (before deduplication).
+   * Map from definition name to count.
+   */
+  parameterQueryResultsByDefinition?: Map<string, number>;
 }
 
 export class BucketParameterState {
@@ -502,11 +530,21 @@ export class BucketParameterState {
         ErrorCode.PSYNC_S2305,
         `Too many parameter query results: ${update.buckets.length} (limit of ${this.context.maxParameterQueryResults})`
       );
-      this.logger.error(error.message, {
+
+      let errorMessage = error.message;
+      const logData: any = {
         checkpoint: checkpoint,
         user_id: this.syncParams.userId,
-        buckets: update.buckets.length
-      });
+        parameter_query_results: update.buckets.length
+      };
+
+      if (update.parameterQueryResultsByDefinition && update.parameterQueryResultsByDefinition.size > 0) {
+        const breakdown = formatParameterQueryBreakdown(update.parameterQueryResultsByDefinition);
+        errorMessage += breakdown.message;
+        logData.parameter_query_results_by_definition = breakdown.countsByDefinition;
+      }
+
+      this.logger.error(errorMessage, logData);
 
       throw error;
     }
@@ -562,6 +600,7 @@ export class BucketParameterState {
     }
 
     let dynamicBuckets: ResolvedBucket[];
+    let parameterQueryResultsByDefinition: Map<string, number> | undefined;
     if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
       const recordedLookups = new Set<string>();
 
@@ -574,6 +613,14 @@ export class BucketParameterState {
           return checkpoint.base.getParameterSets(lookups);
         }
       });
+
+      // Count parameter query results per definition (before deduplication)
+      parameterQueryResultsByDefinition = new Map<string, number>();
+      for (const bucket of dynamicBuckets) {
+        const count = parameterQueryResultsByDefinition.get(bucket.definition) ?? 0;
+        parameterQueryResultsByDefinition.set(bucket.definition, count + 1);
+      }
+
       this.cachedDynamicBuckets = dynamicBuckets;
       this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
       this.lookupsFromPreviousCheckpoint = recordedLookups;
@@ -596,12 +643,14 @@ export class BucketParameterState {
       return {
         buckets: allBuckets,
         // We cannot track individual bucket updates for dynamic lookups yet
-        updatedBuckets: INVALIDATE_ALL_BUCKETS
+        updatedBuckets: INVALIDATE_ALL_BUCKETS,
+        parameterQueryResultsByDefinition
       };
     } else {
       return {
         buckets: allBuckets,
-        updatedBuckets: updatedBuckets
+        updatedBuckets: updatedBuckets,
+        parameterQueryResultsByDefinition
       };
     }
   }
@@ -609,7 +658,7 @@ export class BucketParameterState {
 
 export interface CheckpointLine {
   checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
-  bucketsToFetch: BucketDescription[];
+  bucketsToFetch: ResolvedBucket[];
 
   /**
    * Call when a checkpoint line is being sent to a client, to update the internal state.
@@ -621,7 +670,7 @@ export interface CheckpointLine {
    *
    * @param bucketsToFetch List of buckets to fetch - either this.bucketsToFetch, or a subset of it. Defaults to this.bucketsToFetch.
    */
-  getFilteredBucketPositions(bucketsToFetch?: BucketDescription[]): Map<string, util.InternalOpId>;
+  getFilteredBucketPositions(bucketsToFetch?: ResolvedBucket[]): storage.BucketDataRequest[];
 
   /**
    * Update the position of bucket data the client has, after it was sent to the client.
@@ -635,6 +684,68 @@ export interface CheckpointLine {
 // Use a more specific type to simplify testing
 export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
 
+/**
+ * Compute the total number of parameter query results across all definitions.
+ */
+function computeTotalParamResults(
+  parameterQueryResultsByDefinition: Map<string, number> | undefined
+): number | undefined {
+  if (!parameterQueryResultsByDefinition) {
+    return undefined;
+  }
+  return Array.from(parameterQueryResultsByDefinition.values()).reduce((sum, count) => sum + count, 0);
+}
+
+/**
+ * Log a checkpoint message, enriching it with parameter query result counts if available.
+ *
+ * @param logger The logger instance to use
+ * @param message The base message string (param_results will NOT be appended — caller includes it if needed)
+ * @param logData The base log data object
+ * @param totalParamResults The total parameter query results count, or undefined if not applicable
+ */
+function logCheckpoint(
+  logger: Logger,
+  message: string,
+  logData: Record<string, any>,
+  totalParamResults: number | undefined
+): void {
+  if (totalParamResults !== undefined) {
+    logData.parameter_query_results = totalParamResults;
+  }
+  logger.info(message, logData);
+}
+
+/**
+ * Format a breakdown of parameter query results by sync rule definition.
+ *
+ * Sorts definitions by count (descending), includes the top 10, and returns both the
+ * formatted message string and the counts record suitable for structured log data.
+ */
+function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<string, number>): {
+  message: string;
+  countsByDefinition: Record<string, number>;
+} {
+  // Sort definitions by count (descending) and take top 10
+  const allSorted = Array.from(parameterQueryResultsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
+  const sortedDefinitions = allSorted.slice(0, 10);
+
+  let message = '\nParameter query results by definition:';
+  const countsByDefinition: Record<string, number> = {};
+  for (const [definition, count] of sortedDefinitions) {
+    message += `\n  ${definition}: ${count}`;
+    countsByDefinition[definition] = count;
+  }
+
+  if (allSorted.length > 10) {
+    const remainingResults = allSorted.slice(10).reduce((sum, [, count]) => sum + count, 0);
+    const remainingDefinitions = allSorted.length - 10;
+    message += `\n  ... and ${remainingResults} more results from ${remainingDefinitions} definitions`;
+  }
+
+  return { message, countsByDefinition };
+}
+
 function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
   buckets = buckets.map((b) => {
     if (typeof b != 'string') {
@@ -649,32 +760,3 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number)
   const limited = buckets.slice(0, limit);
   return `${JSON.stringify(limited)}...`;
 }
-
-/**
- * Resolves duplicate buckets in the given array, merging the inclusion reasons for duplicate.
- *
- * It's possible for duplicates to occur when a stream has multiple subscriptions, consider e.g.
- *
- * ```
- * sync_streams:
- *  assets_by_category:
- *    query: select * from assets where category in (request.parameters() -> 'categories')
- * ```
- *
- * Here, a client might subscribe once with `{"categories": [1]}` and once with `{"categories": [1, 2]}`. Since each
- * subscription is evaluated independently, this would lead to three buckets, with a duplicate `assets_by_category[1]`
- * bucket.
- */
-function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] {
-  const byBucketId: Record<string, ResolvedBucket> = {};
-
-  for (const bucket of buckets) {
-    if (Object.hasOwn(byBucketId, bucket.bucket)) {
-      byBucketId[bucket.bucket].inclusion_reasons.push(...bucket.inclusion_reasons);
-    } else {
-      byBucketId[bucket.bucket] = structuredClone(bucket);
-    }
-  }
-
-  return Object.values(byBucketId);
-}

package/src/sync/sync.ts

@@ -1,5 +1,11 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import { BucketDescription, BucketPriority, HydratedSyncRules, SqliteJsonValue } from '@powersync/service-sync-rules';
+import {
+  BucketDescription,
+  BucketPriority,
+  HydratedSyncRules,
+  ResolvedBucket,
+  SqliteJsonValue
+} from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -179,7 +185,7 @@ async function* streamResponseInner(
       // receive a sync complete message after the synchronization is done (which happens in the last
       // bucketDataInBatches iteration). Without any batch, the line is missing and clients might not complete their
       // sync properly.
-      const priorityBatches: [BucketPriority | null, BucketDescription[]][] = bucketsByPriority;
+      const priorityBatches: [BucketPriority | null, ResolvedBucket[]][] = bucketsByPriority;
       if (priorityBatches.length == 0) {
         priorityBatches.push([null, []]);
       }
@@ -257,7 +263,7 @@ interface BucketDataRequest {
   /** Contains current bucket state. Modified by the request as data is sent. */
   checkpointLine: CheckpointLine;
   /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
-  bucketsToFetch: BucketDescription[];
+  bucketsToFetch: ResolvedBucket[];
   /** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
   legacyDataLines: boolean;
   /** Signals that the connection was aborted and that streaming should stop ASAP. */

package/src/sync/util.ts

@@ -183,6 +183,16 @@ export function settledPromise<T>(promise: Promise<T>): Promise<PromiseSettledRe
   );
 }
 
+export function unsettledPromise<T>(settled: Promise<PromiseSettledResult<T>>): Promise<T> {
+  return settled.then((result) => {
+    if (result.status === 'fulfilled') {
+      return Promise.resolve(result.value);
+    } else {
+      return Promise.reject(result.reason);
+    }
+  });
+}
+
 export type MapOrSet<T> = Map<T, any> | Set<T>;
 
 /**

package/src/util/config/collectors/config-collector.ts

@@ -40,6 +40,16 @@ export abstract class ConfigCollector {
      */
     const decoded = this.decode(serialized);
     this.validate(decoded);
+
+    /**
+     * For internal convenience, we duplicate sync_rules and sync_config. Making them interchangeable.
+     * Note, we only do this after validation (which only allows one option to be present)
+     */
+    if (decoded.sync_config) {
+      decoded.sync_rules = decoded.sync_config;
+    } else if (decoded.sync_rules) {
+      decoded.sync_config = decoded.sync_rules;
+    }
     return decoded;
   }
 
@@ -52,6 +62,12 @@ export abstract class ConfigCollector {
     if (!valid.valid) {
       throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
     }
+
+    if (config.sync_config && config.sync_rules) {
+      throw new Error(
+        'Both `sync_config` and `sync_rules` are present in the service configuration. Please consolidate into one sync_config.'
+      );
+    }
   }
 
   decode(encoded: configFile.SerializedPowerSyncConfig): configFile.PowerSyncConfig {

package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts

@@ -1,6 +1,6 @@
+import { configFile } from '@powersync/service-types';
 import { RunnerConfig, SyncRulesConfig } from '../../types.js';
 import { SyncRulesCollector } from '../sync-collector.js';
-import { configFile } from '@powersync/service-types';
 
 export class Base64SyncRulesCollector extends SyncRulesCollector {
   get name(): string {
@@ -8,15 +8,15 @@ export class Base64SyncRulesCollector extends SyncRulesCollector {
   }
 
   async collect(baseConfig: configFile.PowerSyncConfig, runnerConfig: RunnerConfig): Promise<SyncRulesConfig | null> {
-    const { sync_rules_base64 } = runnerConfig;
-    if (!sync_rules_base64) {
+    const { sync_config_base64 } = runnerConfig;
+    if (!sync_config_base64) {
       return null;
     }
 
     return {
       present: true,
-      exit_on_error: baseConfig.sync_rules?.exit_on_error ?? true,
-      content: Buffer.from(sync_rules_base64, 'base64').toString()
+      exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
+      content: Buffer.from(sync_config_base64, 'base64').toString()
     };
   }
 }

package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts

@@ -1,7 +1,7 @@
+import { configFile } from '@powersync/service-types';
 import * as path from 'path';
 import { RunnerConfig, SyncRulesConfig } from '../../types.js';
 import { SyncRulesCollector } from '../sync-collector.js';
-import { configFile } from '@powersync/service-types';
 
 export class FileSystemSyncRulesCollector extends SyncRulesCollector {
   get name(): string {
@@ -9,7 +9,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
   }
 
   async collect(baseConfig: configFile.PowerSyncConfig, runnerConfig: RunnerConfig): Promise<SyncRulesConfig | null> {
-    const sync_path = baseConfig.sync_rules?.path;
+    const sync_path = baseConfig.sync_config?.path;
     if (!sync_path) {
       return null;
     }
@@ -20,7 +20,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
     // Only persist the path here, and load on demand using `loadSyncRules()`.
     return {
       present: true,
-      exit_on_error: baseConfig.sync_rules?.exit_on_error ?? true,
+      exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
       path: config_path ? path.resolve(path.dirname(config_path), sync_path) : sync_path
     };
   }

package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts

@@ -1,6 +1,6 @@
+import { configFile } from '@powersync/service-types';
 import { SyncRulesConfig } from '../../types.js';
 import { SyncRulesCollector } from '../sync-collector.js';
-import { configFile } from '@powersync/service-types';
 
 export class InlineSyncRulesCollector extends SyncRulesCollector {
   get name(): string {
@@ -8,15 +8,15 @@ export class InlineSyncRulesCollector extends SyncRulesCollector {
   }
 
   async collect(baseConfig: configFile.PowerSyncConfig): Promise<SyncRulesConfig | null> {
-    const content = baseConfig.sync_rules?.content;
+    const content = baseConfig?.sync_config?.content;
     if (!content) {
       return null;
     }
 
     return {
       present: true,
-      exit_on_error: true,
-      ...baseConfig.sync_rules
+      exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
+      ...baseConfig.sync_config
     };
   }
 }

package/src/util/config/types.ts

@@ -1,6 +1,5 @@
 import { configFile } from '@powersync/service-types';
 import { CompoundKeyCollector } from '../../auth/CompoundKeyCollector.js';
-import { KeySpec } from '../../auth/KeySpec.js';
 import { KeyStore } from '../../auth/KeyStore.js';
 
 export enum ServiceRunner {
@@ -12,7 +11,7 @@ export enum ServiceRunner {
 export type RunnerConfig = {
   config_path?: string;
   config_base64?: string;
-  sync_rules_base64?: string;
+  sync_config_base64?: string;
 };
 
 export type MigrationContext = {

package/src/util/env.ts

@@ -12,9 +12,14 @@ export const env = utils.collectEnvironmentVariables({
    */
   POWERSYNC_CONFIG_B64: utils.type.string.optional(),
   /**
+   * @deprecated use POWERSYNC_SYNC_CONFIG_B64 instead.
    * Base64 encoded contents of sync rules YAML
    */
   POWERSYNC_SYNC_RULES_B64: utils.type.string.optional(),
+  /**
+   * Base64 encoded contents of sync config YAML
+   */
+  POWERSYNC_SYNC_CONFIG_B64: utils.type.string.optional(),
   /**
    * Runner to be started in this process
    */