@powersync/service-core 1.22.0 → 1.23.0

package/src/storage/implementation/BucketDefinitionMapping.ts

@@ -0,0 +1,436 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  BucketDataSource,
+  BucketDefinitionId,
+  HashMap,
+  ParameterIndexId,
+  ParameterIndexLookupCreator,
+  ParameterLookupDefinitionId,
+  SerializedBucketDataSourceWithDataSources,
+  SerializedParameterIndexLookupCreator,
+  serializedStreamBucketDataSourceEquality,
+  serializedStreamParameterIndexLookupCreatorEquality,
+  SerializedSyncPlanV1,
+  SourceTableRef,
+  SyncConfigWithErrors
+} from '@powersync/service-sync-rules';
+
+export interface SerializedSyncConfigWithMapping {
+  plan: SerializedSyncPlanV1;
+  mapping: SingleSyncConfigBucketDefinitionMapping;
+}
+
+export type IncrementalMappingDefinitionType = 'bucket_data' | 'parameter_lookup';
+
+export interface IncrementalMappingDefinitionChange {
+  type: IncrementalMappingDefinitionType;
+  name: string;
+  id: BucketDefinitionId | ParameterIndexId;
+}
+
+export interface IncrementalMappingChanges {
+  reusedDefinitions: IncrementalMappingDefinitionChange[];
+  addedDefinitions: IncrementalMappingDefinitionChange[];
+}
+
+export interface IncrementalMappingResult {
+  mapping: SingleSyncConfigBucketDefinitionMapping;
+  changes: IncrementalMappingChanges;
+}
+export interface SyncConfigWithMapping {
+  syncConfigId?: string;
+  syncConfig: SyncConfigWithErrors;
+  mapping: SingleSyncConfigBucketDefinitionMapping | null;
+}
+
+export interface SyncConfigWithRequiredMapping {
+  syncConfigId: string;
+  syncConfig: SyncConfigWithErrors;
+  mapping: SingleSyncConfigBucketDefinitionMapping;
+}
+
+export interface PersistedDefinitionMapping {
+  /**
+   * Map of uniqueName -> id, unique per replication stream.
+   */
+  definitions: Record<string, string>;
+  /**
+   * Map of (lookupName, queryId) -> id, unique per replication stream.
+   */
+  parameter_indexes: Record<string, string>;
+}
+
+/**
+ * Represents a mapping from bucket data sources and parameter lookup sources to stable IDs used for bucket definition and parameter index persistence.
+ *
+ * Usage is deliberately restricted: writer/batch paths resolve through their own parsed
+ * sync config set's mapping, and read paths resolve through the persisted
+ * {@link SingleSyncConfigBucketDefinitionMapping} of the single sync config they serve.
+ * Do not construct ad-hoc resolution paths from parsed sources.
+ */
+export interface BucketDefinitionMapping {
+  /**
+   * Given a BucketDataSource within the associated SyncConfig(s), return the BucketDefinitionId, or throw if not found.
+   */
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId;
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId;
+
+  allBucketDefinitionIds(): BucketDefinitionId[];
+
+  allParameterIndexIds(): ParameterIndexId[];
+
+  syncConfigIdsForSourceTable(
+    selectedSyncConfigIds: string[],
+    table: SourceTableRef,
+    bucketDataSourceIds: BucketDefinitionId[],
+    parameterLookupSourceIds: ParameterIndexId[]
+  ): string[];
+
+  snapshotBlockingSourceTablesFilter(syncConfigId: string): Record<string, unknown>;
+}
+
+/**
+ * A BucketDefinitionMapping associated with a single SyncConfig.
+ *
+ * Within a single SyncConfig, unique names are the persistence key for bucket definitions,
+ * so lookups here are name-based by design. MongoHydrationState and
+ * MultiSyncConfigBucketDefinitionMapping handle the mapping across multiple SyncConfigs in
+ * the same replication stream, where names are not unique.
+ */
+export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping {
+  static fromPersistedMapping(mapping: PersistedDefinitionMapping): SingleSyncConfigBucketDefinitionMapping {
+    return new SingleSyncConfigBucketDefinitionMapping(mapping.definitions ?? {}, mapping.parameter_indexes ?? {});
+  }
+
+  static fromParsedSyncConfig(syncConfig: SyncConfigWithErrors): SingleSyncConfigBucketDefinitionMapping {
+    const definitionNames = syncConfig.config.bucketDataSources.map((source) => source.uniqueName).sort();
+    const parameterKeys = syncConfig.config.bucketParameterLookupSources
+      .map((source) => `${source.sourceId.lookupName}#${source.sourceId.queryId}`)
+      .sort();
+
+    const definitions: Record<string, BucketDefinitionId> = {};
+    const parameterLookups: Record<string, ParameterIndexId> = {};
+
+    for (const [index, uniqueName] of definitionNames.entries()) {
+      definitions[uniqueName] = (index + 1).toString(16);
+    }
+    for (const [index, key] of parameterKeys.entries()) {
+      parameterLookups[key] = (index + 1).toString(16);
+    }
+
+    return new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups);
+  }
+
+  /**
+   *
+   * @param compatibleConfigs SyncConfigs with definitions that may be re-used
+   * @param newPlan
+   * @param reservedMappings _all_ mappings used currently and historically for the replication stream; used to construct new ids
+   * @returns
+   */
+  static constructIncrementalMappingFromSerializedPlans(
+    compatibleConfigs: SerializedSyncConfigWithMapping[],
+    newPlan: SerializedSyncPlanV1,
+    reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
+  ): SingleSyncConfigBucketDefinitionMapping {
+    return this.constructIncrementalMappingWithChanges(compatibleConfigs, newPlan, reservedMappings).mapping;
+  }
+
+  static constructIncrementalMappingWithChanges(
+    compatibleConfigs: SerializedSyncConfigWithMapping[],
+    newPlan: SerializedSyncPlanV1,
+    reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
+  ): IncrementalMappingResult {
+    let nextBucketDefinitionId =
+      reservedMappings
+        .map((mapping) => mapping.allBucketDefinitionIds())
+        .flat()
+        .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1;
+    function generateNewBucketDefinitionId(): BucketDefinitionId {
+      const id = nextBucketDefinitionId.toString(16);
+      nextBucketDefinitionId++;
+      return id;
+    }
+    let nextParameterIndexId =
+      reservedMappings
+        .map((mapping) => mapping.allParameterIndexIds())
+        .flat()
+        .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1;
+    function generateNewParameterIndexId(): ParameterIndexId {
+      const id = nextParameterIndexId.toString(16);
+      nextParameterIndexId++;
+      return id;
+    }
+
+    const definitions: Record<string, BucketDefinitionId> = {};
+    const parameterLookups: Record<string, ParameterIndexId> = {};
+    const changes: IncrementalMappingChanges = {
+      reusedDefinitions: [],
+      addedDefinitions: []
+    };
+    const compatibleBuckets = new HashMap<SerializedBucketDataSourceWithDataSources, BucketDefinitionId>(
+      serializedStreamBucketDataSourceEquality
+    );
+    const compatibleParameterLookups = new HashMap<SerializedParameterIndexLookupCreator, ParameterIndexId>(
+      serializedStreamParameterIndexLookupCreatorEquality
+    );
+
+    for (const config of compatibleConfigs) {
+      for (const bucket of config.plan.buckets) {
+        compatibleBuckets.putIfAbsent({ bucket, dataSources: config.plan.dataSources }, () =>
+          config.mapping.bucketSourceIdByName(bucket.uniqueName)
+        );
+      }
+
+      for (const parameterLookup of config.plan.parameterIndexes) {
+        compatibleParameterLookups.putIfAbsent(parameterLookup, () =>
+          config.mapping.parameterLookupIdByKey(parameterLookupKey(parameterLookup.lookupScope))
+        );
+      }
+    }
+
+    for (const bucket of newPlan.buckets) {
+      const compatibleId = compatibleBuckets.get({ bucket, dataSources: newPlan.dataSources });
+      const id = compatibleId ?? generateNewBucketDefinitionId();
+      definitions[bucket.uniqueName] = id;
+      const change: IncrementalMappingDefinitionChange = {
+        type: 'bucket_data',
+        name: bucket.uniqueName,
+        id
+      };
+      if (compatibleId == null) {
+        changes.addedDefinitions.push(change);
+      } else {
+        changes.reusedDefinitions.push(change);
+      }
+    }
+
+    for (const parameterLookup of newPlan.parameterIndexes) {
+      const compatibleId = compatibleParameterLookups.get(parameterLookup);
+      const id = compatibleId ?? generateNewParameterIndexId();
+      const key = parameterLookupKey(parameterLookup.lookupScope);
+      parameterLookups[key] = id;
+      const change: IncrementalMappingDefinitionChange = {
+        type: 'parameter_lookup',
+        name: key,
+        id
+      };
+      if (compatibleId == null) {
+        changes.addedDefinitions.push(change);
+      } else {
+        changes.reusedDefinitions.push(change);
+      }
+    }
+
+    return {
+      mapping: new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups),
+      changes
+    };
+  }
+
+  constructor(
+    private definitions: Record<string, BucketDefinitionId> = {},
+    private parameterLookupMapping: Record<string, ParameterIndexId> = {}
+  ) {}
+
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId {
+    return this.bucketSourceIdByName(source.uniqueName);
+  }
+
+  bucketSourceIdByName(uniqueName: string): BucketDefinitionId {
+    const defId = this.definitions[uniqueName];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for bucket source ${uniqueName}`);
+    }
+    return defId;
+  }
+
+  allBucketDefinitionIds(): BucketDefinitionId[] {
+    return Object.values(this.definitions);
+  }
+
+  bucketDefinitionEntries(): IncrementalMappingDefinitionChange[] {
+    return Object.entries(this.definitions).map(([name, id]) => ({
+      type: 'bucket_data',
+      name,
+      id
+    }));
+  }
+
+  allParameterIndexIds(): ParameterIndexId[] {
+    return Object.values(this.parameterLookupMapping);
+  }
+
+  parameterIndexEntries(): IncrementalMappingDefinitionChange[] {
+    return Object.entries(this.parameterLookupMapping).map(([name, id]) => ({
+      type: 'parameter_lookup',
+      name,
+      id
+    }));
+  }
+
+  allDefinitionEntries(): IncrementalMappingDefinitionChange[] {
+    return [...this.bucketDefinitionEntries(), ...this.parameterIndexEntries()];
+  }
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId {
+    return this.parameterLookupIdByKey(parameterLookupKey(source.sourceId));
+  }
+
+  parameterLookupIdByKey(key: string): ParameterIndexId {
+    const defId = this.parameterLookupMapping[key];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`);
+    }
+    return defId;
+  }
+
+  serialize(): PersistedDefinitionMapping {
+    return {
+      definitions: { ...this.definitions },
+      parameter_indexes: { ...this.parameterLookupMapping }
+    };
+  }
+
+  syncConfigIdsForSourceTable(
+    selectedSyncConfigIds: string[],
+    _table: SourceTableRef,
+    bucketDataSourceIds: BucketDefinitionId[],
+    parameterLookupSourceIds: ParameterIndexId[]
+  ): string[] {
+    return bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0 ? selectedSyncConfigIds : [];
+  }
+
+  snapshotBlockingSourceTablesFilter(_syncConfigId: string): Record<string, unknown> {
+    const clauses: Record<string, unknown>[] = [];
+    const bucketDataSourceIds = this.allBucketDefinitionIds();
+    if (bucketDataSourceIds.length > 0) {
+      clauses.push({ bucket_data_source_ids: { $in: bucketDataSourceIds } });
+    }
+    const parameterLookupSourceIds = this.allParameterIndexIds();
+    if (parameterLookupSourceIds.length > 0) {
+      clauses.push({ parameter_lookup_source_ids: { $in: parameterLookupSourceIds } });
+    }
+    if (clauses.length == 0) {
+      return {
+        snapshot_done: false,
+        _id: { $exists: false }
+      };
+    }
+    return {
+      snapshot_done: false,
+      $or: clauses
+    };
+  }
+}
+
+/**
+ * A BucketDefinitionMapping across all SyncConfigs of one parse of a replication stream.
+ *
+ * Source lookups are strictly identity-based: a source that did not come from the parse
+ * this mapping was constructed with is an error. Unique names are not unique across sync
+ * configs in a replication stream, so there is deliberately no name-based fallback.
+ */
+export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping {
+  private bucketDataSourceMappings = new WeakMap<BucketDataSource, SingleSyncConfigBucketDefinitionMapping>();
+  private bucketDataSourceSyncConfigIdsById = new Map<BucketDefinitionId, Set<string>>();
+  private parameterLookupMappings = new WeakMap<ParameterIndexLookupCreator, SingleSyncConfigBucketDefinitionMapping>();
+  private parameterLookupSyncConfigIdsById = new Map<ParameterIndexId, Set<string>>();
+  private syncConfigsById = new Map<string, SyncConfigWithRequiredMapping>();
+  private mappings: SingleSyncConfigBucketDefinitionMapping[];
+
+  constructor(syncConfigs: SyncConfigWithRequiredMapping[]) {
+    this.mappings = syncConfigs.map((config) => config.mapping);
+
+    for (const config of syncConfigs) {
+      this.syncConfigsById.set(config.syncConfigId, config);
+      for (const source of config.syncConfig.config.bucketDataSources) {
+        this.bucketDataSourceMappings.set(source, config.mapping);
+        addSetEntry(this.bucketDataSourceSyncConfigIdsById, config.mapping.bucketSourceId(source), config.syncConfigId);
+      }
+      for (const source of config.syncConfig.config.bucketParameterLookupSources) {
+        this.parameterLookupMappings.set(source, config.mapping);
+        addSetEntry(
+          this.parameterLookupSyncConfigIdsById,
+          config.mapping.parameterLookupId(source),
+          config.syncConfigId
+        );
+      }
+    }
+  }
+
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId {
+    const mapping = this.bucketDataSourceMappings.get(source);
+    if (mapping == null) {
+      throw new ServiceAssertionError(
+        `Bucket source ${source.uniqueName} is not associated with this parse of the sync configs`
+      );
+    }
+    return mapping.bucketSourceId(source);
+  }
+
+  allBucketDefinitionIds(): BucketDefinitionId[] {
+    return [...new Set(this.mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))];
+  }
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId {
+    const mapping = this.parameterLookupMappings.get(source);
+    if (mapping == null) {
+      throw new ServiceAssertionError(
+        `Parameter lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId} is not associated with this parse of the sync configs`
+      );
+    }
+    return mapping.parameterLookupId(source);
+  }
+
+  allParameterIndexIds(): ParameterIndexId[] {
+    return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))];
+  }
+
+  syncConfigIdsForSourceTable(
+    _selectedSyncConfigIds: string[],
+    table: SourceTableRef,
+    bucketDataSourceIds: BucketDefinitionId[],
+    parameterLookupSourceIds: ParameterIndexId[]
+  ): string[] {
+    const ids = new Set<string>();
+    for (const sourceId of bucketDataSourceIds) {
+      addAll(ids, this.bucketDataSourceSyncConfigIdsById.get(sourceId));
+    }
+    for (const sourceId of parameterLookupSourceIds) {
+      addAll(ids, this.parameterLookupSyncConfigIdsById.get(sourceId));
+    }
+    for (const [syncConfigId, config] of this.syncConfigsById) {
+      if (config.syncConfig.config.tableTriggersEvent(table)) {
+        ids.add(syncConfigId);
+      }
+    }
+    return [...ids];
+  }
+
+  snapshotBlockingSourceTablesFilter(syncConfigId: string): Record<string, unknown> {
+    const config = this.syncConfigsById.get(syncConfigId);
+    if (config == null) {
+      throw new ServiceAssertionError(`No mapping found for sync config ${syncConfigId}`);
+    }
+
+    return config.mapping.snapshotBlockingSourceTablesFilter(syncConfigId);
+  }
+}
+
+export function parameterLookupKey(id: ParameterLookupDefinitionId) {
+  return `${id.lookupName}#${id.queryId}`;
+}
+
+function addSetEntry<K, V>(map: Map<K, Set<V>>, key: K, value: V) {
+  const existing = map.get(key) ?? new Set<V>();
+  existing.add(value);
+  map.set(key, existing);
+}
+
+function addAll<T>(target: Set<T>, values: Iterable<T> | undefined) {
+  for (const value of values ?? []) {
+    target.add(value);
+  }
+}

package/src/storage/implementation/IncrementalReprocessingCompatibility.ts

@@ -0,0 +1,41 @@
+import { CompatibilityContext } from '@powersync/service-sync-rules';
+import { Logger } from 'winston';
+import { SerializedSyncPlan, UpdateSyncRulesOptions } from '../BucketStorageFactory.js';
+
+/**
+ * Check if a sync config update can use incremental reprocessing.
+ */
+export function isCompatible(
+  existingPlans: (SerializedSyncPlan | null)[],
+  updateConfig: UpdateSyncRulesOptions['config'],
+  logger: Logger
+): boolean {
+  if (updateConfig.plan == null) {
+    // Only support sync streams with serialized plans
+    logger.info(`Not using current sync streams - incremental reprocessing not supported`);
+    return false;
+  }
+
+  // We don't check the storage version here - that is checked upstream.
+  if (existingPlans.length == 0) {
+    // Could technically be compatible, but there is no reason to re-use this stream.
+    return false;
+  }
+
+  if (existingPlans.some((plan) => plan == null)) {
+    // Only support sync streams with serialized plans
+    logger.info(`Existing replication stream not using current sync streams - incremental reprocessing not supported`);
+    return false;
+  }
+
+  // Technically we can compare the serialized compatibility versions? But this does not add much overhead.
+  const first = existingPlans[0];
+  const streamCompatibility = CompatibilityContext.deserialize(first!.compatibility);
+  if (!streamCompatibility.equals(updateConfig.parsed.config.compatibility)) {
+    // Compatibility options must match
+    logger.info(`Compatibility options changed - incremental reprocessing not supported`);
+    return false;
+  }
+
+  return true;
+}

package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts

@@ -0,0 +1,109 @@
+import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+import {
+  IncrementalMappingChanges,
+  IncrementalMappingDefinitionChange,
+  SingleSyncConfigBucketDefinitionMapping
+} from './BucketDefinitionMapping.js';
+
+export interface IncrementalDefinitionLogEntry extends IncrementalMappingDefinitionChange {
+  sourceTables?: string[];
+}
+
+export interface IncrementalSyncConfigUpdateLog {
+  reusedDefinitions: IncrementalDefinitionLogEntry[];
+  addedDefinitions: IncrementalDefinitionLogEntry[];
+  droppedDefinitions: IncrementalDefinitionLogEntry[];
+}
+
+export function formatIncrementalSyncConfigUpdateLog(log: IncrementalSyncConfigUpdateLog) {
+  return [
+    formatDefinitionSection('Reused definitions', log.reusedDefinitions),
+    formatDefinitionSection('New definitions', log.addedDefinitions, { includeSnapshotTables: true }),
+    formatDefinitionSection('Definitions to drop after switching', log.droppedDefinitions)
+  ].join('\n');
+}
+
+export function describeIncrementalSyncConfigUpdate(options: {
+  activeMappings: SingleSyncConfigBucketDefinitionMapping[];
+  newMapping: SingleSyncConfigBucketDefinitionMapping;
+  newSyncConfig: SyncConfigWithErrors;
+  mappingChanges: IncrementalMappingChanges;
+}): IncrementalSyncConfigUpdateLog {
+  const { activeMappings, newMapping, newSyncConfig, mappingChanges } = options;
+  const newDefinitionKeys = new Set(newMapping.allDefinitionEntries().map(definitionKey));
+  const activeDefinitions = uniqueDefinitions(activeMappings.flatMap((mapping) => mapping.allDefinitionEntries()));
+
+  return {
+    reusedDefinitions: mappingChanges.reusedDefinitions,
+    addedDefinitions: mappingChanges.addedDefinitions.map((definition) => {
+      const sourceTables = sourceTablesForDefinition(newSyncConfig, newMapping, definition);
+      return {
+        ...definition,
+        sourceTables
+      };
+    }),
+    droppedDefinitions: activeDefinitions.filter((definition) => !newDefinitionKeys.has(definitionKey(definition)))
+  };
+}
+
+function sourceTablesForDefinition(
+  syncConfig: SyncConfigWithErrors,
+  mapping: SingleSyncConfigBucketDefinitionMapping,
+  definition: IncrementalMappingDefinitionChange
+) {
+  if (definition.type == 'bucket_data') {
+    return sourceTablesForSources(
+      syncConfig.config.bucketDataSources.filter((source) => mapping.bucketSourceId(source) == definition.id)
+    );
+  }
+
+  return sourceTablesForSources(
+    syncConfig.config.bucketParameterLookupSources.filter(
+      (source) => mapping.parameterLookupId(source) == definition.id
+    )
+  );
+}
+
+function sourceTablesForSources(sources: Array<BucketDataSource | ParameterIndexLookupCreator>) {
+  return uniqueSorted(sources.flatMap((source) => [...source.getSourceTables()].map((table) => table.tablePattern)));
+}
+
+function definitionKey(definition: IncrementalMappingDefinitionChange) {
+  return `${definition.type}:${definition.id}`;
+}
+
+function uniqueSorted(values: string[]) {
+  return [...new Set(values)].sort();
+}
+
+function uniqueDefinitions(definitions: IncrementalMappingDefinitionChange[]) {
+  const byKey = new Map<string, IncrementalMappingDefinitionChange>();
+  for (const definition of definitions) {
+    byKey.set(definitionKey(definition), definition);
+  }
+  return [...byKey.values()];
+}
+
+function formatDefinitionSection(
+  title: string,
+  definitions: IncrementalDefinitionLogEntry[],
+  options: { includeSnapshotTables?: boolean } = {}
+) {
+  if (definitions.length == 0) {
+    return `${title}: none`;
+  }
+
+  return [`${title}:`, ...definitions.map((definition) => `  - ${formatDefinition(definition, options)}`)].join('\n');
+}
+
+function formatDefinition(definition: IncrementalDefinitionLogEntry, options: { includeSnapshotTables?: boolean }) {
+  const details = [`type=${definition.type}`, `id=${definition.id}`, `name=${definition.name}`];
+  if (definition.sourceTables != null) {
+    details.push(`tables=${formatList(definition.sourceTables)}`);
+  }
+  return details.join(', ');
+}
+
+function formatList(values: string[]) {
+  return values.length == 0 ? 'none' : values.join(', ');
+}

package/src/storage/implementation/ReplicationStreamStorageIds.ts

@@ -0,0 +1,21 @@
+import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
+import { SingleSyncConfigBucketDefinitionMapping } from './BucketDefinitionMapping.js';
+
+/**
+ * Persisted storage ids for a replication stream, derived from the rule_mapping documents
+ * of all its sync configs.
+ *
+ * This requires no parsed sync configs, and deliberately does not resolve individual
+ * sources - use the parsed-set-bound {@link BucketDefinitionMapping} for that.
+ */
+export class ReplicationStreamStorageIds {
+  constructor(private readonly mappings: SingleSyncConfigBucketDefinitionMapping[]) {}
+
+  get bucketDefinitionIds(): BucketDefinitionId[] {
+    return [...new Set(this.mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))];
+  }
+
+  get parameterIndexIds(): ParameterIndexId[] {
+    return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))];
+  }
+}

package/src/storage/storage-index.ts

@@ -3,7 +3,10 @@ export * from './BucketStorage.js';
 export * from './BucketStorageBatch.js';
 export * from './BucketStorageFactory.js';
 export * from './ChecksumCache.js';
-export * from './PersistedSyncRulesContent.js';
+export * from './ParsedSyncConfigSet.js';
+export * from './PersistedReplicationStream.js';
+export * from './PersistedSyncConfigContent.js';
+export * from './PersistedSyncConfigStatus.js';
 export * from './ReplicationEventPayload.js';
 export * from './ReplicationLock.js';
 export * from './ReportStorage.js';
@@ -15,3 +18,8 @@ export * from './StorageProvider.js';
 export * from './StorageVersionConfig.js';
 export * from './SyncRulesBucketStorage.js';
 export * from './WriteCheckpointAPI.js';
+
+export * from './implementation/BucketDefinitionMapping.js';
+export * from './implementation/IncrementalReprocessingCompatibility.js';
+export * from './implementation/IncrementalReprocessingSyncConfigLog.js';
+export * from './implementation/ReplicationStreamStorageIds.js';

package/src/sync/sync.ts

@@ -367,6 +367,11 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         checkpointInvalidated = true;
       }
       if (r.data.length == 0) {
+        // An empty chunk is not sent to the client, but may still report progress:
+        // storage can emit an empty chunk for a bucket that has no operations in the
+        // requested range, and its position must advance so the bucket is not
+        // re-requested indefinitely.
+        checkpointLine.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
         continue;
       }
       logger.debug(`Sending data for ${r.bucket}`);

package/src/util/checkpointing.ts

@@ -11,7 +11,7 @@ export interface CreateWriteCheckpointOptions {
 export async function createWriteCheckpoint(options: CreateWriteCheckpointOptions) {
   const full_user_id = checkpointUserId(options.userId, options.clientId);
 
-  const syncBucketStorage = await options.storage.getActiveStorage();
+  const syncBucketStorage = (await options.storage.getActiveSyncConfig())?.storage;
   if (!syncBucketStorage) {
     throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
   }

package/src/util/config/types.ts

@@ -55,7 +55,7 @@ export type ResolvedPowerSyncConfig = {
     max_parameter_query_results: number;
   };
 
-  /** Prefix for postgres replication slot names. May eventually be connection-specific. */
+  /** Prefix for postgres replication slot names, as well as replication stream names. */
   slot_name_prefix: string;
 
   healthcheck: {

package/src/util/lsn.ts

@@ -6,3 +6,12 @@ export function maxLsn(a: string | null | undefined, b: string | null | undefine
   if (b == null) return a;
   return a > b ? a : b;
 }
+
+/**
+ * Return the smaller of two LSNs, ignoring nulls.
+ */
+export function minLsn(a: string | null | undefined, b: string | null | undefined): string | null {
+  if (a == null) return b ?? null;
+  if (b == null) return a;
+  return a < b ? a : b;
+}