@powersync/service-core 1.21.0 → 1.23.0

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,17 +1,20 @@
 import { Logger, ObserverClient } from '@powersync/lib-services-framework';
 import {
   BucketDataSource,
-  HydratedSyncRules,
+  HydratedSyncConfig,
   ParameterLookupRows,
   ScopedParameterLookup
 } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
-import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
+import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
+import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
+import { StorageVersionConfig } from './StorageVersionConfig.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
 /**
@@ -20,17 +23,13 @@ import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 export interface SyncRulesBucketStorage
   extends ObserverClient<SyncRulesBucketStorageListener>,
     SyncStorageWriteCheckpointAPI {
-  readonly group_id: number;
-  readonly slot_name: string;
+  readonly replicationStreamId: number;
+  readonly replicationStreamName: string;
+  readonly storageConfig: StorageVersionConfig;
 
   readonly factory: BucketStorageFactory;
   readonly logger: Logger;
 
-  /**
-   * Resolve a table, keeping track of it internally.
-   */
-  resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
-
   /**
    * Create a new writer.
    *
@@ -46,7 +45,20 @@ export interface SyncRulesBucketStorage
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
-  getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules;
+  /**
+   * Get the canonical parsed sync config set for the given parse options.
+   *
+   * Repeated calls with the same options return the same instance for the lifetime of
+   * this storage instance. This is the identity boundary for parsed sync config state:
+   * all operations against this storage instance must use parsed objects from sets
+   * returned here, never from an independent parse of the same content.
+   */
+  getParsedSyncConfigSet(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
+
+  /**
+   * Shortcut for `getParsedSyncConfigSet(options).hydratedSyncConfig`.
+   */
+  getParsedSyncRules(options: ParseSyncConfigOptions): HydratedSyncConfig;
 
   /**
    * Terminate the replication stream.
@@ -57,7 +69,7 @@ export interface SyncRulesBucketStorage
    */
   terminate(options?: TerminateOptions): Promise<void>;
 
-  getStatus(): Promise<SyncRuleStatus>;
+  getStatus(): Promise<ReplicationStreamStatus>;
 
   /**
    * Clear the storage, without changing state.
@@ -135,6 +147,12 @@ export interface SyncRulesBucketStorage
    * Clear checksum cache. Primarily intended for tests.
    */
   clearChecksumCache(): void;
+
+  /**
+   * Optional storage-provider cleanup for sync configs that have been stopped
+   * inside this replication stream.
+   */
+  cleanupStoppedSyncConfigs?(options: CleanupStoppedSyncConfigsOptions): Promise<CleanupStoppedSyncConfigsResult>;
 }
 
 export interface SyncRulesBucketStorageListener {
@@ -151,27 +169,35 @@ export interface BucketChecksumRequest {
   source: BucketDataSource;
 }
 
-export interface SyncRuleStatus {
-  checkpoint_lsn: string | null;
-  active: boolean;
-  snapshot_done: boolean;
-  snapshot_lsn: string | null;
+export interface ReplicationStreamStatus {
+  /**
+   * Source position to resume replication from.
+   */
+  resumeLsn: string | null;
+  /**
+   * True if _every_ active/processing sync config has a client-visible checkpoint.
+   */
+  snapshotDone: boolean;
 }
-export interface ResolveTableOptions {
-  group_id: number;
+export interface ResolveTablesOptions {
   connection_id: number;
-  connection_tag: string;
-  entity_descriptor: SourceEntityDescriptor;
-
-  sync_rules: HydratedSyncRules;
+  source: SourceEntityDescriptor;
+  /**
+   * For tests only - custom id generator for stable ids.
+   */
+  idGenerator?: () => string | bson.ObjectId;
+  /**
+   * For tests only - override the parsed sync config set used.
+   */
+  parsedSyncConfig?: ParsedSyncConfigSet;
 }
 
-export interface ResolveTableResult {
-  table: SourceTable;
+export interface ResolveTablesResult {
+  tables: SourceTable[];
   dropTables: SourceTable[];
 }
 
-export interface CreateWriterOptions extends ParseSyncRulesOptions {
+export interface CreateWriterOptions extends ParseSyncConfigOptions {
   zeroLSN: string;
   /**
    * Whether or not to store a copy of the current data.
@@ -198,11 +224,18 @@ export interface CreateWriterOptions extends ParseSyncRulesOptions {
    */
   markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
 
+  hooks?: StorageHooks;
+
   tracer?: PerformanceTracer<'storage' | 'evaluate'>;
 
   logger?: Logger;
 }
 
+export interface StorageHooks {
+  beforeBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+  afterBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+}
+
 /**
  * @deprecated Use `CreateWriterOptions`.
  */
@@ -249,6 +282,9 @@ export interface CompactOptions {
   /** Minimum of 1 */
   moveBatchQueryLimit?: number;
 
+  /** Byte cap per read batch for streaming compaction. Default 16MB. */
+  moveBatchByteLimit?: number;
+
   /**
    * Minimum number new operations in a bucket to trigger compaction of that bucket.
    *
@@ -274,7 +310,14 @@ export interface CompactOptions {
 }
 
 export interface PopulateChecksumCacheOptions {
-  maxOpId: util.InternalOpId;
+  /**
+   * Compute checksums up to this op id.
+   *
+   * Defaults to the highest persisted op id for the replication stream, which covers
+   * the common case of populating the cache right after initial replication, before
+   * the first checkpoint exists.
+   */
+  maxOpId?: util.InternalOpId;
   minBucketChanges?: number;
   signal?: AbortSignal;
 }
@@ -290,6 +333,24 @@ export interface ClearStorageOptions {
   signal?: AbortSignal;
 }
 
+export interface CleanupStoppedSyncConfigsOptions {
+  signal?: AbortSignal;
+  logger?: Logger;
+  defaultSchema: string;
+  sourceConnectionTag: string;
+}
+
+export interface CleanupStoppedSyncConfigsResult {
+  stoppedSyncConfigsRemoved: number;
+  bucketDataCollectionsDropped: number;
+  parameterIndexCollectionsDropped: number;
+  /** Best-effort number, not accurate if the operation takes longer than the timeout. */
+  bucketStateDocumentsDeleted: number;
+  sourceRecordCollectionsDropped: number;
+  sourceTablesUpdated: number;
+  sourceTablesDeleted: number;
+}
+
 export interface TerminateOptions extends ClearStorageOptions {
   /**
    * If true, also clear the storage before terminating.

package/src/storage/implementation/BucketDefinitionMapping.ts

@@ -0,0 +1,436 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  BucketDataSource,
+  BucketDefinitionId,
+  HashMap,
+  ParameterIndexId,
+  ParameterIndexLookupCreator,
+  ParameterLookupDefinitionId,
+  SerializedBucketDataSourceWithDataSources,
+  SerializedParameterIndexLookupCreator,
+  serializedStreamBucketDataSourceEquality,
+  serializedStreamParameterIndexLookupCreatorEquality,
+  SerializedSyncPlanV1,
+  SourceTableRef,
+  SyncConfigWithErrors
+} from '@powersync/service-sync-rules';
+
+export interface SerializedSyncConfigWithMapping {
+  plan: SerializedSyncPlanV1;
+  mapping: SingleSyncConfigBucketDefinitionMapping;
+}
+
+export type IncrementalMappingDefinitionType = 'bucket_data' | 'parameter_lookup';
+
+export interface IncrementalMappingDefinitionChange {
+  type: IncrementalMappingDefinitionType;
+  name: string;
+  id: BucketDefinitionId | ParameterIndexId;
+}
+
+export interface IncrementalMappingChanges {
+  reusedDefinitions: IncrementalMappingDefinitionChange[];
+  addedDefinitions: IncrementalMappingDefinitionChange[];
+}
+
+export interface IncrementalMappingResult {
+  mapping: SingleSyncConfigBucketDefinitionMapping;
+  changes: IncrementalMappingChanges;
+}
+export interface SyncConfigWithMapping {
+  syncConfigId?: string;
+  syncConfig: SyncConfigWithErrors;
+  mapping: SingleSyncConfigBucketDefinitionMapping | null;
+}
+
+export interface SyncConfigWithRequiredMapping {
+  syncConfigId: string;
+  syncConfig: SyncConfigWithErrors;
+  mapping: SingleSyncConfigBucketDefinitionMapping;
+}
+
+export interface PersistedDefinitionMapping {
+  /**
+   * Map of uniqueName -> id, unique per replication stream.
+   */
+  definitions: Record<string, string>;
+  /**
+   * Map of (lookupName, queryId) -> id, unique per replication stream.
+   */
+  parameter_indexes: Record<string, string>;
+}
+
+/**
+ * Represents a mapping from bucket data sources and parameter lookup sources to stable IDs used for bucket definition and parameter index persistence.
+ *
+ * Usage is deliberately restricted: writer/batch paths resolve through their own parsed
+ * sync config set's mapping, and read paths resolve through the persisted
+ * {@link SingleSyncConfigBucketDefinitionMapping} of the single sync config they serve.
+ * Do not construct ad-hoc resolution paths from parsed sources.
+ */
+export interface BucketDefinitionMapping {
+  /**
+   * Given a BucketDataSource within the associated SyncConfig(s), return the BucketDefinitionId, or throw if not found.
+   */
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId;
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId;
+
+  allBucketDefinitionIds(): BucketDefinitionId[];
+
+  allParameterIndexIds(): ParameterIndexId[];
+
+  syncConfigIdsForSourceTable(
+    selectedSyncConfigIds: string[],
+    table: SourceTableRef,
+    bucketDataSourceIds: BucketDefinitionId[],
+    parameterLookupSourceIds: ParameterIndexId[]
+  ): string[];
+
+  snapshotBlockingSourceTablesFilter(syncConfigId: string): Record<string, unknown>;
+}
+
+/**
+ * A BucketDefinitionMapping associated with a single SyncConfig.
+ *
+ * Within a single SyncConfig, unique names are the persistence key for bucket definitions,
+ * so lookups here are name-based by design. MongoHydrationState and
+ * MultiSyncConfigBucketDefinitionMapping handle the mapping across multiple SyncConfigs in
+ * the same replication stream, where names are not unique.
+ */
+export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping {
+  static fromPersistedMapping(mapping: PersistedDefinitionMapping): SingleSyncConfigBucketDefinitionMapping {
+    return new SingleSyncConfigBucketDefinitionMapping(mapping.definitions ?? {}, mapping.parameter_indexes ?? {});
+  }
+
+  static fromParsedSyncConfig(syncConfig: SyncConfigWithErrors): SingleSyncConfigBucketDefinitionMapping {
+    const definitionNames = syncConfig.config.bucketDataSources.map((source) => source.uniqueName).sort();
+    const parameterKeys = syncConfig.config.bucketParameterLookupSources
+      .map((source) => `${source.sourceId.lookupName}#${source.sourceId.queryId}`)
+      .sort();
+
+    const definitions: Record<string, BucketDefinitionId> = {};
+    const parameterLookups: Record<string, ParameterIndexId> = {};
+
+    for (const [index, uniqueName] of definitionNames.entries()) {
+      definitions[uniqueName] = (index + 1).toString(16);
+    }
+    for (const [index, key] of parameterKeys.entries()) {
+      parameterLookups[key] = (index + 1).toString(16);
+    }
+
+    return new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups);
+  }
+
+  /**
+   *
+   * @param compatibleConfigs SyncConfigs with definitions that may be re-used
+   * @param newPlan
+   * @param reservedMappings _all_ mappings used currently and historically for the replication stream; used to construct new ids
+   * @returns
+   */
+  static constructIncrementalMappingFromSerializedPlans(
+    compatibleConfigs: SerializedSyncConfigWithMapping[],
+    newPlan: SerializedSyncPlanV1,
+    reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
+  ): SingleSyncConfigBucketDefinitionMapping {
+    return this.constructIncrementalMappingWithChanges(compatibleConfigs, newPlan, reservedMappings).mapping;
+  }
+
+  static constructIncrementalMappingWithChanges(
+    compatibleConfigs: SerializedSyncConfigWithMapping[],
+    newPlan: SerializedSyncPlanV1,
+    reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
+  ): IncrementalMappingResult {
+    let nextBucketDefinitionId =
+      reservedMappings
+        .map((mapping) => mapping.allBucketDefinitionIds())
+        .flat()
+        .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1;
+    function generateNewBucketDefinitionId(): BucketDefinitionId {
+      const id = nextBucketDefinitionId.toString(16);
+      nextBucketDefinitionId++;
+      return id;
+    }
+    let nextParameterIndexId =
+      reservedMappings
+        .map((mapping) => mapping.allParameterIndexIds())
+        .flat()
+        .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1;
+    function generateNewParameterIndexId(): ParameterIndexId {
+      const id = nextParameterIndexId.toString(16);
+      nextParameterIndexId++;
+      return id;
+    }
+
+    const definitions: Record<string, BucketDefinitionId> = {};
+    const parameterLookups: Record<string, ParameterIndexId> = {};
+    const changes: IncrementalMappingChanges = {
+      reusedDefinitions: [],
+      addedDefinitions: []
+    };
+    const compatibleBuckets = new HashMap<SerializedBucketDataSourceWithDataSources, BucketDefinitionId>(
+      serializedStreamBucketDataSourceEquality
+    );
+    const compatibleParameterLookups = new HashMap<SerializedParameterIndexLookupCreator, ParameterIndexId>(
+      serializedStreamParameterIndexLookupCreatorEquality
+    );
+
+    for (const config of compatibleConfigs) {
+      for (const bucket of config.plan.buckets) {
+        compatibleBuckets.putIfAbsent({ bucket, dataSources: config.plan.dataSources }, () =>
+          config.mapping.bucketSourceIdByName(bucket.uniqueName)
+        );
+      }
+
+      for (const parameterLookup of config.plan.parameterIndexes) {
+        compatibleParameterLookups.putIfAbsent(parameterLookup, () =>
+          config.mapping.parameterLookupIdByKey(parameterLookupKey(parameterLookup.lookupScope))
+        );
+      }
+    }
+
+    for (const bucket of newPlan.buckets) {
+      const compatibleId = compatibleBuckets.get({ bucket, dataSources: newPlan.dataSources });
+      const id = compatibleId ?? generateNewBucketDefinitionId();
+      definitions[bucket.uniqueName] = id;
+      const change: IncrementalMappingDefinitionChange = {
+        type: 'bucket_data',
+        name: bucket.uniqueName,
+        id
+      };
+      if (compatibleId == null) {
+        changes.addedDefinitions.push(change);
+      } else {
+        changes.reusedDefinitions.push(change);
+      }
+    }
+
+    for (const parameterLookup of newPlan.parameterIndexes) {
+      const compatibleId = compatibleParameterLookups.get(parameterLookup);
+      const id = compatibleId ?? generateNewParameterIndexId();
+      const key = parameterLookupKey(parameterLookup.lookupScope);
+      parameterLookups[key] = id;
+      const change: IncrementalMappingDefinitionChange = {
+        type: 'parameter_lookup',
+        name: key,
+        id
+      };
+      if (compatibleId == null) {
+        changes.addedDefinitions.push(change);
+      } else {
+        changes.reusedDefinitions.push(change);
+      }
+    }
+
+    return {
+      mapping: new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups),
+      changes
+    };
+  }
+
+  constructor(
+    private definitions: Record<string, BucketDefinitionId> = {},
+    private parameterLookupMapping: Record<string, ParameterIndexId> = {}
+  ) {}
+
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId {
+    return this.bucketSourceIdByName(source.uniqueName);
+  }
+
+  bucketSourceIdByName(uniqueName: string): BucketDefinitionId {
+    const defId = this.definitions[uniqueName];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for bucket source ${uniqueName}`);
+    }
+    return defId;
+  }
+
+  allBucketDefinitionIds(): BucketDefinitionId[] {
+    return Object.values(this.definitions);
+  }
+
+  bucketDefinitionEntries(): IncrementalMappingDefinitionChange[] {
+    return Object.entries(this.definitions).map(([name, id]) => ({
+      type: 'bucket_data',
+      name,
+      id
+    }));
+  }
+
+  allParameterIndexIds(): ParameterIndexId[] {
+    return Object.values(this.parameterLookupMapping);
+  }
+
+  parameterIndexEntries(): IncrementalMappingDefinitionChange[] {
+    return Object.entries(this.parameterLookupMapping).map(([name, id]) => ({
+      type: 'parameter_lookup',
+      name,
+      id
+    }));
+  }
+
+  allDefinitionEntries(): IncrementalMappingDefinitionChange[] {
+    return [...this.bucketDefinitionEntries(), ...this.parameterIndexEntries()];
+  }
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId {
+    return this.parameterLookupIdByKey(parameterLookupKey(source.sourceId));
+  }
+
+  parameterLookupIdByKey(key: string): ParameterIndexId {
+    const defId = this.parameterLookupMapping[key];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`);
+    }
+    return defId;
+  }
+
+  serialize(): PersistedDefinitionMapping {
+    return {
+      definitions: { ...this.definitions },
+      parameter_indexes: { ...this.parameterLookupMapping }
+    };
+  }
+
+  syncConfigIdsForSourceTable(
+    selectedSyncConfigIds: string[],
+    _table: SourceTableRef,
+    bucketDataSourceIds: BucketDefinitionId[],
+    parameterLookupSourceIds: ParameterIndexId[]
+  ): string[] {
+    return bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0 ? selectedSyncConfigIds : [];
+  }
+
+  snapshotBlockingSourceTablesFilter(_syncConfigId: string): Record<string, unknown> {
+    const clauses: Record<string, unknown>[] = [];
+    const bucketDataSourceIds = this.allBucketDefinitionIds();
+    if (bucketDataSourceIds.length > 0) {
+      clauses.push({ bucket_data_source_ids: { $in: bucketDataSourceIds } });
+    }
+    const parameterLookupSourceIds = this.allParameterIndexIds();
+    if (parameterLookupSourceIds.length > 0) {
+      clauses.push({ parameter_lookup_source_ids: { $in: parameterLookupSourceIds } });
+    }
+    if (clauses.length == 0) {
+      return {
+        snapshot_done: false,
+        _id: { $exists: false }
+      };
+    }
+    return {
+      snapshot_done: false,
+      $or: clauses
+    };
+  }
+}
+
+/**
+ * A BucketDefinitionMapping across all SyncConfigs of one parse of a replication stream.
+ *
+ * Source lookups are strictly identity-based: a source that did not come from the parse
+ * this mapping was constructed with is an error. Unique names are not unique across sync
+ * configs in a replication stream, so there is deliberately no name-based fallback.
+ */
+export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping {
+  private bucketDataSourceMappings = new WeakMap<BucketDataSource, SingleSyncConfigBucketDefinitionMapping>();
+  private bucketDataSourceSyncConfigIdsById = new Map<BucketDefinitionId, Set<string>>();
+  private parameterLookupMappings = new WeakMap<ParameterIndexLookupCreator, SingleSyncConfigBucketDefinitionMapping>();
+  private parameterLookupSyncConfigIdsById = new Map<ParameterIndexId, Set<string>>();
+  private syncConfigsById = new Map<string, SyncConfigWithRequiredMapping>();
+  private mappings: SingleSyncConfigBucketDefinitionMapping[];
+
+  constructor(syncConfigs: SyncConfigWithRequiredMapping[]) {
+    this.mappings = syncConfigs.map((config) => config.mapping);
+
+    for (const config of syncConfigs) {
+      this.syncConfigsById.set(config.syncConfigId, config);
+      for (const source of config.syncConfig.config.bucketDataSources) {
+        this.bucketDataSourceMappings.set(source, config.mapping);
+        addSetEntry(this.bucketDataSourceSyncConfigIdsById, config.mapping.bucketSourceId(source), config.syncConfigId);
+      }
+      for (const source of config.syncConfig.config.bucketParameterLookupSources) {
+        this.parameterLookupMappings.set(source, config.mapping);
+        addSetEntry(
+          this.parameterLookupSyncConfigIdsById,
+          config.mapping.parameterLookupId(source),
+          config.syncConfigId
+        );
+      }
+    }
+  }
+
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId {
+    const mapping = this.bucketDataSourceMappings.get(source);
+    if (mapping == null) {
+      throw new ServiceAssertionError(
+        `Bucket source ${source.uniqueName} is not associated with this parse of the sync configs`
+      );
+    }
+    return mapping.bucketSourceId(source);
+  }
+
+  allBucketDefinitionIds(): BucketDefinitionId[] {
+    return [...new Set(this.mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))];
+  }
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId {
+    const mapping = this.parameterLookupMappings.get(source);
+    if (mapping == null) {
+      throw new ServiceAssertionError(
+        `Parameter lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId} is not associated with this parse of the sync configs`
+      );
+    }
+    return mapping.parameterLookupId(source);
+  }
+
+  allParameterIndexIds(): ParameterIndexId[] {
+    return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))];
+  }
+
+  syncConfigIdsForSourceTable(
+    _selectedSyncConfigIds: string[],
+    table: SourceTableRef,
+    bucketDataSourceIds: BucketDefinitionId[],
+    parameterLookupSourceIds: ParameterIndexId[]
+  ): string[] {
+    const ids = new Set<string>();
+    for (const sourceId of bucketDataSourceIds) {
+      addAll(ids, this.bucketDataSourceSyncConfigIdsById.get(sourceId));
+    }
+    for (const sourceId of parameterLookupSourceIds) {
+      addAll(ids, this.parameterLookupSyncConfigIdsById.get(sourceId));
+    }
+    for (const [syncConfigId, config] of this.syncConfigsById) {
+      if (config.syncConfig.config.tableTriggersEvent(table)) {
+        ids.add(syncConfigId);
+      }
+    }
+    return [...ids];
+  }
+
+  snapshotBlockingSourceTablesFilter(syncConfigId: string): Record<string, unknown> {
+    const config = this.syncConfigsById.get(syncConfigId);
+    if (config == null) {
+      throw new ServiceAssertionError(`No mapping found for sync config ${syncConfigId}`);
+    }
+
+    return config.mapping.snapshotBlockingSourceTablesFilter(syncConfigId);
+  }
+}
+
+export function parameterLookupKey(id: ParameterLookupDefinitionId) {
+  return `${id.lookupName}#${id.queryId}`;
+}
+
+function addSetEntry<K, V>(map: Map<K, Set<V>>, key: K, value: V) {
+  const existing = map.get(key) ?? new Set<V>();
+  existing.add(value);
+  map.set(key, existing);
+}
+
+function addAll<T>(target: Set<T>, values: Iterable<T> | undefined) {
+  for (const value of values ?? []) {
+    target.add(value);
+  }
+}

package/src/storage/implementation/IncrementalReprocessingCompatibility.ts

@@ -0,0 +1,41 @@
+import { CompatibilityContext } from '@powersync/service-sync-rules';
+import { Logger } from 'winston';
+import { SerializedSyncPlan, UpdateSyncRulesOptions } from '../BucketStorageFactory.js';
+
+/**
+ * Check if a sync config update can use incremental reprocessing.
+ */
+export function isCompatible(
+  existingPlans: (SerializedSyncPlan | null)[],
+  updateConfig: UpdateSyncRulesOptions['config'],
+  logger: Logger
+): boolean {
+  if (updateConfig.plan == null) {
+    // Only support sync streams with serialized plans
+    logger.info(`Not using current sync streams - incremental reprocessing not supported`);
+    return false;
+  }
+
+  // We don't check the storage version here - that is checked upstream.
+  if (existingPlans.length == 0) {
+    // Could technically be compatible, but there is no reason to re-use this stream.
+    return false;
+  }
+
+  if (existingPlans.some((plan) => plan == null)) {
+    // Only support sync streams with serialized plans
+    logger.info(`Existing replication stream not using current sync streams - incremental reprocessing not supported`);
+    return false;
+  }
+
+  // Technically we can compare the serialized compatibility versions? But this does not add much overhead.
+  const first = existingPlans[0];
+  const streamCompatibility = CompatibilityContext.deserialize(first!.compatibility);
+  if (!streamCompatibility.equals(updateConfig.parsed.config.compatibility)) {
+    // Compatibility options must match
+    logger.info(`Compatibility options changed - incremental reprocessing not supported`);
+    return false;
+  }
+
+  return true;
+}