@powersync/service-core 1.21.0 → 1.23.0

package/dist/storage/SyncRulesBucketStorage.d.ts

@@ -1,25 +1,25 @@
 import { Logger, ObserverClient } from '@powersync/lib-services-framework';
-import { BucketDataSource, HydratedSyncRules, ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
+import { BucketDataSource, HydratedSyncConfig, ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
-import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
+import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
+import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
+import { StorageVersionConfig } from './StorageVersionConfig.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 /**
  * Storage for a specific replication stream.
  */
 export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketStorageListener>, SyncStorageWriteCheckpointAPI {
-    readonly group_id: number;
-    readonly slot_name: string;
+    readonly replicationStreamId: number;
+    readonly replicationStreamName: string;
+    readonly storageConfig: StorageVersionConfig;
     readonly factory: BucketStorageFactory;
     readonly logger: Logger;
-    /**
-     * Resolve a table, keeping track of it internally.
-     */
-    resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
     /**
      * Create a new writer.
      *
@@ -30,7 +30,19 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
      * @deprecated Use `createWriter()` with `await using` instead.
      */
     startBatch(options: CreateWriterOptions, callback: (batch: BucketStorageBatch) => Promise<void>): Promise<FlushedResult | null>;
-    getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules;
+    /**
+     * Get the canonical parsed sync config set for the given parse options.
+     *
+     * Repeated calls with the same options return the same instance for the lifetime of
+     * this storage instance. This is the identity boundary for parsed sync config state:
+     * all operations against this storage instance must use parsed objects from sets
+     * returned here, never from an independent parse of the same content.
+     */
+    getParsedSyncConfigSet(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
+    /**
+     * Shortcut for `getParsedSyncConfigSet(options).hydratedSyncConfig`.
+     */
+    getParsedSyncRules(options: ParseSyncConfigOptions): HydratedSyncConfig;
     /**
      * Terminate the replication stream.
      *
@@ -39,7 +51,7 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
      * Must only be called on stopped replication streams.
      */
     terminate(options?: TerminateOptions): Promise<void>;
-    getStatus(): Promise<SyncRuleStatus>;
+    getStatus(): Promise<ReplicationStreamStatus>;
     /**
      * Clear the storage, without changing state.
      */
@@ -101,6 +113,11 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
      * Clear checksum cache. Primarily intended for tests.
      */
     clearChecksumCache(): void;
+    /**
+     * Optional storage-provider cleanup for sync configs that have been stopped
+     * inside this replication stream.
+     */
+    cleanupStoppedSyncConfigs?(options: CleanupStoppedSyncConfigsOptions): Promise<CleanupStoppedSyncConfigsResult>;
 }
 export interface SyncRulesBucketStorageListener {
     batchStarted: (batch: BucketStorageBatch) => void;
@@ -114,24 +131,33 @@ export interface BucketChecksumRequest {
     bucket: string;
     source: BucketDataSource;
 }
-export interface SyncRuleStatus {
-    checkpoint_lsn: string | null;
-    active: boolean;
-    snapshot_done: boolean;
-    snapshot_lsn: string | null;
+export interface ReplicationStreamStatus {
+    /**
+     * Source position to resume replication from.
+     */
+    resumeLsn: string | null;
+    /**
+     * True if _every_ active/processing sync config has a client-visible checkpoint.
+     */
+    snapshotDone: boolean;
 }
-export interface ResolveTableOptions {
-    group_id: number;
+export interface ResolveTablesOptions {
     connection_id: number;
-    connection_tag: string;
-    entity_descriptor: SourceEntityDescriptor;
-    sync_rules: HydratedSyncRules;
+    source: SourceEntityDescriptor;
+    /**
+     * For tests only - custom id generator for stable ids.
+     */
+    idGenerator?: () => string | bson.ObjectId;
+    /**
+     * For tests only - override the parsed sync config set used.
+     */
+    parsedSyncConfig?: ParsedSyncConfigSet;
 }
-export interface ResolveTableResult {
-    table: SourceTable;
+export interface ResolveTablesResult {
+    tables: SourceTable[];
     dropTables: SourceTable[];
 }
-export interface CreateWriterOptions extends ParseSyncRulesOptions {
+export interface CreateWriterOptions extends ParseSyncConfigOptions {
     zeroLSN: string;
     /**
      * Whether or not to store a copy of the current data.
@@ -155,9 +181,14 @@ export interface CreateWriterOptions extends ParseSyncRulesOptions {
      * This is expected to happen in some initial replication edge cases, only if storeCurrentData = true.
      */
     markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
+    hooks?: StorageHooks;
     tracer?: PerformanceTracer<'storage' | 'evaluate'>;
     logger?: Logger;
 }
+export interface StorageHooks {
+    beforeBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+    afterBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+}
 /**
  * @deprecated Use `CreateWriterOptions`.
  */
@@ -197,6 +228,8 @@ export interface CompactOptions {
     moveBatchLimit?: number;
     /** Minimum of 1 */
     moveBatchQueryLimit?: number;
+    /** Byte cap per read batch for streaming compaction. Default 16MB. */
+    moveBatchByteLimit?: number;
     /**
      * Minimum number new operations in a bucket to trigger compaction of that bucket.
      *
@@ -217,7 +250,14 @@ export interface CompactOptions {
     logger?: Logger;
 }
 export interface PopulateChecksumCacheOptions {
-    maxOpId: util.InternalOpId;
+    /**
+     * Compute checksums up to this op id.
+     *
+     * Defaults to the highest persisted op id for the replication stream, which covers
+     * the common case of populating the cache right after initial replication, before
+     * the first checkpoint exists.
+     */
+    maxOpId?: util.InternalOpId;
     minBucketChanges?: number;
     signal?: AbortSignal;
 }
@@ -230,6 +270,22 @@ export interface PopulateChecksumCacheResults {
 export interface ClearStorageOptions {
     signal?: AbortSignal;
 }
+export interface CleanupStoppedSyncConfigsOptions {
+    signal?: AbortSignal;
+    logger?: Logger;
+    defaultSchema: string;
+    sourceConnectionTag: string;
+}
+export interface CleanupStoppedSyncConfigsResult {
+    stoppedSyncConfigsRemoved: number;
+    bucketDataCollectionsDropped: number;
+    parameterIndexCollectionsDropped: number;
+    /** Best-effort number, not accurate if the operation takes longer than the timeout. */
+    bucketStateDocumentsDeleted: number;
+    sourceRecordCollectionsDropped: number;
+    sourceTablesUpdated: number;
+    sourceTablesDeleted: number;
+}
 export interface TerminateOptions extends ClearStorageOptions {
     /**
      * If true, also clear the storage before terminating.

package/dist/storage/SyncRulesBucketStorage.js.map

@@ -1 +1 @@
-{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AA+UA;;;;GAIG;AACH,MAAM,OAAO,8BAA+B,SAAQ,KAAK;IAE5C;IACA;IAFX,YACW,KAAa,EACb,SAAyC;QAElD,KAAK,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;QAHhD,UAAK,GAAL,KAAK,CAAQ;QACb,cAAS,GAAT,SAAS,CAAgC;IAGpD,CAAC;CACF;AAqDD,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,IAAI,GAAG,EAAU;IACrC,qBAAqB,EAAE,IAAI;IAC3B,uBAAuB,EAAE,IAAI,GAAG,EAAU;IAC1C,0BAA0B,EAAE,IAAI;CACjC,CAAC"}
+{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AA4YA;;;;GAIG;AACH,MAAM,OAAO,8BAA+B,SAAQ,KAAK;IAE5C;IACA;IAFX,YACW,KAAa,EACb,SAAyC;QAElD,KAAK,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;QAHhD,UAAK,GAAL,KAAK,CAAQ;QACb,cAAS,GAAT,SAAS,CAAgC;IAGpD,CAAC;CACF;AAqDD,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,IAAI,GAAG,EAAU;IACrC,qBAAqB,EAAE,IAAI;IAC3B,uBAAuB,EAAE,IAAI,GAAG,EAAU;IAC1C,0BAA0B,EAAE,IAAI;CACjC,CAAC"}

package/dist/storage/implementation/BucketDefinitionMapping.d.ts

@@ -0,0 +1,117 @@
+import { BucketDataSource, BucketDefinitionId, ParameterIndexId, ParameterIndexLookupCreator, ParameterLookupDefinitionId, SerializedSyncPlanV1, SourceTableRef, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+export interface SerializedSyncConfigWithMapping {
+    plan: SerializedSyncPlanV1;
+    mapping: SingleSyncConfigBucketDefinitionMapping;
+}
+export type IncrementalMappingDefinitionType = 'bucket_data' | 'parameter_lookup';
+export interface IncrementalMappingDefinitionChange {
+    type: IncrementalMappingDefinitionType;
+    name: string;
+    id: BucketDefinitionId | ParameterIndexId;
+}
+export interface IncrementalMappingChanges {
+    reusedDefinitions: IncrementalMappingDefinitionChange[];
+    addedDefinitions: IncrementalMappingDefinitionChange[];
+}
+export interface IncrementalMappingResult {
+    mapping: SingleSyncConfigBucketDefinitionMapping;
+    changes: IncrementalMappingChanges;
+}
+export interface SyncConfigWithMapping {
+    syncConfigId?: string;
+    syncConfig: SyncConfigWithErrors;
+    mapping: SingleSyncConfigBucketDefinitionMapping | null;
+}
+export interface SyncConfigWithRequiredMapping {
+    syncConfigId: string;
+    syncConfig: SyncConfigWithErrors;
+    mapping: SingleSyncConfigBucketDefinitionMapping;
+}
+export interface PersistedDefinitionMapping {
+    /**
+     * Map of uniqueName -> id, unique per replication stream.
+     */
+    definitions: Record<string, string>;
+    /**
+     * Map of (lookupName, queryId) -> id, unique per replication stream.
+     */
+    parameter_indexes: Record<string, string>;
+}
+/**
+ * Represents a mapping from bucket data sources and parameter lookup sources to stable IDs used for bucket definition and parameter index persistence.
+ *
+ * Usage is deliberately restricted: writer/batch paths resolve through their own parsed
+ * sync config set's mapping, and read paths resolve through the persisted
+ * {@link SingleSyncConfigBucketDefinitionMapping} of the single sync config they serve.
+ * Do not construct ad-hoc resolution paths from parsed sources.
+ */
+export interface BucketDefinitionMapping {
+    /**
+     * Given a BucketDataSource within the associated SyncConfig(s), return the BucketDefinitionId, or throw if not found.
+     */
+    bucketSourceId(source: BucketDataSource): BucketDefinitionId;
+    parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId;
+    allBucketDefinitionIds(): BucketDefinitionId[];
+    allParameterIndexIds(): ParameterIndexId[];
+    syncConfigIdsForSourceTable(selectedSyncConfigIds: string[], table: SourceTableRef, bucketDataSourceIds: BucketDefinitionId[], parameterLookupSourceIds: ParameterIndexId[]): string[];
+    snapshotBlockingSourceTablesFilter(syncConfigId: string): Record<string, unknown>;
+}
+/**
+ * A BucketDefinitionMapping associated with a single SyncConfig.
+ *
+ * Within a single SyncConfig, unique names are the persistence key for bucket definitions,
+ * so lookups here are name-based by design. MongoHydrationState and
+ * MultiSyncConfigBucketDefinitionMapping handle the mapping across multiple SyncConfigs in
+ * the same replication stream, where names are not unique.
+ */
+export declare class SingleSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping {
+    private definitions;
+    private parameterLookupMapping;
+    static fromPersistedMapping(mapping: PersistedDefinitionMapping): SingleSyncConfigBucketDefinitionMapping;
+    static fromParsedSyncConfig(syncConfig: SyncConfigWithErrors): SingleSyncConfigBucketDefinitionMapping;
+    /**
+     *
+     * @param compatibleConfigs SyncConfigs with definitions that may be re-used
+     * @param newPlan
+     * @param reservedMappings _all_ mappings used currently and historically for the replication stream; used to construct new ids
+     * @returns
+     */
+    static constructIncrementalMappingFromSerializedPlans(compatibleConfigs: SerializedSyncConfigWithMapping[], newPlan: SerializedSyncPlanV1, reservedMappings: SingleSyncConfigBucketDefinitionMapping[]): SingleSyncConfigBucketDefinitionMapping;
+    static constructIncrementalMappingWithChanges(compatibleConfigs: SerializedSyncConfigWithMapping[], newPlan: SerializedSyncPlanV1, reservedMappings: SingleSyncConfigBucketDefinitionMapping[]): IncrementalMappingResult;
+    constructor(definitions?: Record<string, BucketDefinitionId>, parameterLookupMapping?: Record<string, ParameterIndexId>);
+    bucketSourceId(source: BucketDataSource): BucketDefinitionId;
+    bucketSourceIdByName(uniqueName: string): BucketDefinitionId;
+    allBucketDefinitionIds(): BucketDefinitionId[];
+    bucketDefinitionEntries(): IncrementalMappingDefinitionChange[];
+    allParameterIndexIds(): ParameterIndexId[];
+    parameterIndexEntries(): IncrementalMappingDefinitionChange[];
+    allDefinitionEntries(): IncrementalMappingDefinitionChange[];
+    parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId;
+    parameterLookupIdByKey(key: string): ParameterIndexId;
+    serialize(): PersistedDefinitionMapping;
+    syncConfigIdsForSourceTable(selectedSyncConfigIds: string[], _table: SourceTableRef, bucketDataSourceIds: BucketDefinitionId[], parameterLookupSourceIds: ParameterIndexId[]): string[];
+    snapshotBlockingSourceTablesFilter(_syncConfigId: string): Record<string, unknown>;
+}
+/**
+ * A BucketDefinitionMapping across all SyncConfigs of one parse of a replication stream.
+ *
+ * Source lookups are strictly identity-based: a source that did not come from the parse
+ * this mapping was constructed with is an error. Unique names are not unique across sync
+ * configs in a replication stream, so there is deliberately no name-based fallback.
+ */
+export declare class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping {
+    private bucketDataSourceMappings;
+    private bucketDataSourceSyncConfigIdsById;
+    private parameterLookupMappings;
+    private parameterLookupSyncConfigIdsById;
+    private syncConfigsById;
+    private mappings;
+    constructor(syncConfigs: SyncConfigWithRequiredMapping[]);
+    bucketSourceId(source: BucketDataSource): BucketDefinitionId;
+    allBucketDefinitionIds(): BucketDefinitionId[];
+    parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId;
+    allParameterIndexIds(): ParameterIndexId[];
+    syncConfigIdsForSourceTable(_selectedSyncConfigIds: string[], table: SourceTableRef, bucketDataSourceIds: BucketDefinitionId[], parameterLookupSourceIds: ParameterIndexId[]): string[];
+    snapshotBlockingSourceTablesFilter(syncConfigId: string): Record<string, unknown>;
+}
+export declare function parameterLookupKey(id: ParameterLookupDefinitionId): string;

package/dist/storage/implementation/BucketDefinitionMapping.js

@@ -0,0 +1,277 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { HashMap, serializedStreamBucketDataSourceEquality, serializedStreamParameterIndexLookupCreatorEquality } from '@powersync/service-sync-rules';
+/**
+ * A BucketDefinitionMapping associated with a single SyncConfig.
+ *
+ * Within a single SyncConfig, unique names are the persistence key for bucket definitions,
+ * so lookups here are name-based by design. MongoHydrationState and
+ * MultiSyncConfigBucketDefinitionMapping handle the mapping across multiple SyncConfigs in
+ * the same replication stream, where names are not unique.
+ */
+export class SingleSyncConfigBucketDefinitionMapping {
+    definitions;
+    parameterLookupMapping;
+    static fromPersistedMapping(mapping) {
+        return new SingleSyncConfigBucketDefinitionMapping(mapping.definitions ?? {}, mapping.parameter_indexes ?? {});
+    }
+    static fromParsedSyncConfig(syncConfig) {
+        const definitionNames = syncConfig.config.bucketDataSources.map((source) => source.uniqueName).sort();
+        const parameterKeys = syncConfig.config.bucketParameterLookupSources
+            .map((source) => `${source.sourceId.lookupName}#${source.sourceId.queryId}`)
+            .sort();
+        const definitions = {};
+        const parameterLookups = {};
+        for (const [index, uniqueName] of definitionNames.entries()) {
+            definitions[uniqueName] = (index + 1).toString(16);
+        }
+        for (const [index, key] of parameterKeys.entries()) {
+            parameterLookups[key] = (index + 1).toString(16);
+        }
+        return new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups);
+    }
+    /**
+     *
+     * @param compatibleConfigs SyncConfigs with definitions that may be re-used
+     * @param newPlan
+     * @param reservedMappings _all_ mappings used currently and historically for the replication stream; used to construct new ids
+     * @returns
+     */
+    static constructIncrementalMappingFromSerializedPlans(compatibleConfigs, newPlan, reservedMappings) {
+        return this.constructIncrementalMappingWithChanges(compatibleConfigs, newPlan, reservedMappings).mapping;
+    }
+    static constructIncrementalMappingWithChanges(compatibleConfigs, newPlan, reservedMappings) {
+        let nextBucketDefinitionId = reservedMappings
+            .map((mapping) => mapping.allBucketDefinitionIds())
+            .flat()
+            .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1;
+        function generateNewBucketDefinitionId() {
+            const id = nextBucketDefinitionId.toString(16);
+            nextBucketDefinitionId++;
+            return id;
+        }
+        let nextParameterIndexId = reservedMappings
+            .map((mapping) => mapping.allParameterIndexIds())
+            .flat()
+            .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1;
+        function generateNewParameterIndexId() {
+            const id = nextParameterIndexId.toString(16);
+            nextParameterIndexId++;
+            return id;
+        }
+        const definitions = {};
+        const parameterLookups = {};
+        const changes = {
+            reusedDefinitions: [],
+            addedDefinitions: []
+        };
+        const compatibleBuckets = new HashMap(serializedStreamBucketDataSourceEquality);
+        const compatibleParameterLookups = new HashMap(serializedStreamParameterIndexLookupCreatorEquality);
+        for (const config of compatibleConfigs) {
+            for (const bucket of config.plan.buckets) {
+                compatibleBuckets.putIfAbsent({ bucket, dataSources: config.plan.dataSources }, () => config.mapping.bucketSourceIdByName(bucket.uniqueName));
+            }
+            for (const parameterLookup of config.plan.parameterIndexes) {
+                compatibleParameterLookups.putIfAbsent(parameterLookup, () => config.mapping.parameterLookupIdByKey(parameterLookupKey(parameterLookup.lookupScope)));
+            }
+        }
+        for (const bucket of newPlan.buckets) {
+            const compatibleId = compatibleBuckets.get({ bucket, dataSources: newPlan.dataSources });
+            const id = compatibleId ?? generateNewBucketDefinitionId();
+            definitions[bucket.uniqueName] = id;
+            const change = {
+                type: 'bucket_data',
+                name: bucket.uniqueName,
+                id
+            };
+            if (compatibleId == null) {
+                changes.addedDefinitions.push(change);
+            }
+            else {
+                changes.reusedDefinitions.push(change);
+            }
+        }
+        for (const parameterLookup of newPlan.parameterIndexes) {
+            const compatibleId = compatibleParameterLookups.get(parameterLookup);
+            const id = compatibleId ?? generateNewParameterIndexId();
+            const key = parameterLookupKey(parameterLookup.lookupScope);
+            parameterLookups[key] = id;
+            const change = {
+                type: 'parameter_lookup',
+                name: key,
+                id
+            };
+            if (compatibleId == null) {
+                changes.addedDefinitions.push(change);
+            }
+            else {
+                changes.reusedDefinitions.push(change);
+            }
+        }
+        return {
+            mapping: new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups),
+            changes
+        };
+    }
+    constructor(definitions = {}, parameterLookupMapping = {}) {
+        this.definitions = definitions;
+        this.parameterLookupMapping = parameterLookupMapping;
+    }
+    bucketSourceId(source) {
+        return this.bucketSourceIdByName(source.uniqueName);
+    }
+    bucketSourceIdByName(uniqueName) {
+        const defId = this.definitions[uniqueName];
+        if (defId == null) {
+            throw new ServiceAssertionError(`No mapping found for bucket source ${uniqueName}`);
+        }
+        return defId;
+    }
+    allBucketDefinitionIds() {
+        return Object.values(this.definitions);
+    }
+    bucketDefinitionEntries() {
+        return Object.entries(this.definitions).map(([name, id]) => ({
+            type: 'bucket_data',
+            name,
+            id
+        }));
+    }
+    allParameterIndexIds() {
+        return Object.values(this.parameterLookupMapping);
+    }
+    parameterIndexEntries() {
+        return Object.entries(this.parameterLookupMapping).map(([name, id]) => ({
+            type: 'parameter_lookup',
+            name,
+            id
+        }));
+    }
+    allDefinitionEntries() {
+        return [...this.bucketDefinitionEntries(), ...this.parameterIndexEntries()];
+    }
+    parameterLookupId(source) {
+        return this.parameterLookupIdByKey(parameterLookupKey(source.sourceId));
+    }
+    parameterLookupIdByKey(key) {
+        const defId = this.parameterLookupMapping[key];
+        if (defId == null) {
+            throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`);
+        }
+        return defId;
+    }
+    serialize() {
+        return {
+            definitions: { ...this.definitions },
+            parameter_indexes: { ...this.parameterLookupMapping }
+        };
+    }
+    syncConfigIdsForSourceTable(selectedSyncConfigIds, _table, bucketDataSourceIds, parameterLookupSourceIds) {
+        return bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0 ? selectedSyncConfigIds : [];
+    }
+    snapshotBlockingSourceTablesFilter(_syncConfigId) {
+        const clauses = [];
+        const bucketDataSourceIds = this.allBucketDefinitionIds();
+        if (bucketDataSourceIds.length > 0) {
+            clauses.push({ bucket_data_source_ids: { $in: bucketDataSourceIds } });
+        }
+        const parameterLookupSourceIds = this.allParameterIndexIds();
+        if (parameterLookupSourceIds.length > 0) {
+            clauses.push({ parameter_lookup_source_ids: { $in: parameterLookupSourceIds } });
+        }
+        if (clauses.length == 0) {
+            return {
+                snapshot_done: false,
+                _id: { $exists: false }
+            };
+        }
+        return {
+            snapshot_done: false,
+            $or: clauses
+        };
+    }
+}
+/**
+ * A BucketDefinitionMapping across all SyncConfigs of one parse of a replication stream.
+ *
+ * Source lookups are strictly identity-based: a source that did not come from the parse
+ * this mapping was constructed with is an error. Unique names are not unique across sync
+ * configs in a replication stream, so there is deliberately no name-based fallback.
+ */
+export class MultiSyncConfigBucketDefinitionMapping {
+    bucketDataSourceMappings = new WeakMap();
+    bucketDataSourceSyncConfigIdsById = new Map();
+    parameterLookupMappings = new WeakMap();
+    parameterLookupSyncConfigIdsById = new Map();
+    syncConfigsById = new Map();
+    mappings;
+    constructor(syncConfigs) {
+        this.mappings = syncConfigs.map((config) => config.mapping);
+        for (const config of syncConfigs) {
+            this.syncConfigsById.set(config.syncConfigId, config);
+            for (const source of config.syncConfig.config.bucketDataSources) {
+                this.bucketDataSourceMappings.set(source, config.mapping);
+                addSetEntry(this.bucketDataSourceSyncConfigIdsById, config.mapping.bucketSourceId(source), config.syncConfigId);
+            }
+            for (const source of config.syncConfig.config.bucketParameterLookupSources) {
+                this.parameterLookupMappings.set(source, config.mapping);
+                addSetEntry(this.parameterLookupSyncConfigIdsById, config.mapping.parameterLookupId(source), config.syncConfigId);
+            }
+        }
+    }
+    bucketSourceId(source) {
+        const mapping = this.bucketDataSourceMappings.get(source);
+        if (mapping == null) {
+            throw new ServiceAssertionError(`Bucket source ${source.uniqueName} is not associated with this parse of the sync configs`);
+        }
+        return mapping.bucketSourceId(source);
+    }
+    allBucketDefinitionIds() {
+        return [...new Set(this.mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))];
+    }
+    parameterLookupId(source) {
+        const mapping = this.parameterLookupMappings.get(source);
+        if (mapping == null) {
+            throw new ServiceAssertionError(`Parameter lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId} is not associated with this parse of the sync configs`);
+        }
+        return mapping.parameterLookupId(source);
+    }
+    allParameterIndexIds() {
+        return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))];
+    }
+    syncConfigIdsForSourceTable(_selectedSyncConfigIds, table, bucketDataSourceIds, parameterLookupSourceIds) {
+        const ids = new Set();
+        for (const sourceId of bucketDataSourceIds) {
+            addAll(ids, this.bucketDataSourceSyncConfigIdsById.get(sourceId));
+        }
+        for (const sourceId of parameterLookupSourceIds) {
+            addAll(ids, this.parameterLookupSyncConfigIdsById.get(sourceId));
+        }
+        for (const [syncConfigId, config] of this.syncConfigsById) {
+            if (config.syncConfig.config.tableTriggersEvent(table)) {
+                ids.add(syncConfigId);
+            }
+        }
+        return [...ids];
+    }
+    snapshotBlockingSourceTablesFilter(syncConfigId) {
+        const config = this.syncConfigsById.get(syncConfigId);
+        if (config == null) {
+            throw new ServiceAssertionError(`No mapping found for sync config ${syncConfigId}`);
+        }
+        return config.mapping.snapshotBlockingSourceTablesFilter(syncConfigId);
+    }
+}
+export function parameterLookupKey(id) {
+    return `${id.lookupName}#${id.queryId}`;
+}
+function addSetEntry(map, key, value) {
+    const existing = map.get(key) ?? new Set();
+    existing.add(value);
+    map.set(key, existing);
+}
+function addAll(target, values) {
+    for (const value of values ?? []) {
+        target.add(value);
+    }
+}
+//# sourceMappingURL=BucketDefinitionMapping.js.map

package/dist/storage/implementation/BucketDefinitionMapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BucketDefinitionMapping.js","sourceRoot":"","sources":["../../../src/storage/implementation/BucketDefinitionMapping.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAGL,OAAO,EAMP,wCAAwC,EACxC,mDAAmD,EAIpD,MAAM,+BAA+B,CAAC;AA6EvC;;;;;;;GAOG;AACH,MAAM,OAAO,uCAAuC;IAoIxC;IACA;IApIV,MAAM,CAAC,oBAAoB,CAAC,OAAmC;QAC7D,OAAO,IAAI,uCAAuC,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,MAAM,CAAC,oBAAoB,CAAC,UAAgC;QAC1D,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;QACtG,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,4BAA4B;aACjE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;aAC3E,IAAI,EAAE,CAAC;QAEV,MAAM,WAAW,GAAuC,EAAE,CAAC;QAC3D,MAAM,gBAAgB,GAAqC,EAAE,CAAC;QAE9D,KAAK,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,eAAe,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5D,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,IAAI,uCAAuC,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,8CAA8C,CACnD,iBAAoD,EACpD,OAA6B,EAC7B,gBAA2D;QAE3D,OAAO,IAAI,CAAC,sCAAsC,CAAC,iBAAiB,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC,OAAO,CAAC;IAC3G,CAAC;IAED,MAAM,CAAC,sCAAsC,CAC3C,iBAAoD,EACpD,OAA6B,EAC7B,gBAA2D;QAE3D,IAAI,sBAAsB,GACxB,gBAAgB;aACb,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;aAClD,IAAI,EAAE;aACN,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,SAAS,6BAA6B;YACpC,MAAM,EAAE,GAAG,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC/C,sBAAsB,EAAE,CAAC;YACzB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,IAAI,oBAAoB,GACtB,gBAAgB;aACb,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;aAChD,IAAI,EAAE;aACN,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,SAAS,2BAA2B;YAClC,MAAM,EAAE,GAAG,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC7C,oBAAoB,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,WAAW,GAAuC,EAAE,CAAC;QAC3D,MAAM,gBAAgB,GAAqC,EAAE,CAAC;QAC9D,MAAM,OAAO,GAA8B;YACzC,iBAAiB,EAAE,EAAE;YACrB,gBAAgB,EAAE,EAAE;SACrB,CAAC;QACF,MAAM,iBAAiB,GAAG,IAAI,OAAO,CACnC,wCAAwC,CACzC,CAAC;QACF,MAAM,0BAA0B,GAAG,IAAI,OAAO,CAC5C,mDAAmD,CACpD,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;YACvC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACzC,iBAAiB,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CACnF,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,MAAM,CAAC,UAAU,CAAC,CACvD,CAAC;YACJ,CAAC;YAED,KAAK,MAAM,eAAe,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC3D,0BAA0B,CAAC,WAAW,CAAC,eAAe,EAAE,GAAG,EAAE,CAC3D,MAAM,CAAC,OAAO,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CACvF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,iBAAiB,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;YACzF,MAAM,EAAE,GAAG,YAAY,IAAI,6BAA6B,EAAE,CAAC;YAC3D,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,MAAM,GAAuC;gBACjD,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,MAAM,CAAC,UAAU;gBACvB,EAAE;aACH,CAAC;YACF,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;gBACzB,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,eAAe,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACvD,MAAM,YAAY,GAAG,0BAA0B,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;YACrE,MAAM,EAAE,GAAG,YAAY,IAAI,2BAA2B,EAAE,CAAC;YACzD,MAAM,GAAG,GAAG,kBAAkB,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YAC5D,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAuC;gBACjD,IAAI,EAAE,kBAAkB;gBACxB,IAAI,EAAE,GAAG;gBACT,EAAE;aACH,CAAC;YACF,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;gBACzB,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI,uCAAuC,CAAC,WAAW,EAAE,gBAAgB,CAAC;YACnF,OAAO;SACR,CAAC;IACJ,CAAC;IAED,YACU,cAAkD,EAAE,EACpD,yBAA2D,EAAE;QAD7D,gBAAW,GAAX,WAAW,CAAyC;QACpD,2BAAsB,GAAtB,sBAAsB,CAAuC;IACpE,CAAC;IAEJ,cAAc,CAAC,MAAwB;QACrC,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAED,oBAAoB,CAAC,UAAkB;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,qBAAqB,CAAC,sCAAsC,UAAU,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sBAAsB;QACpB,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzC,CAAC;IAED,uBAAuB;QACrB,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI,EAAE,aAAa;YACnB,IAAI;YACJ,EAAE;SACH,CAAC,CAAC,CAAC;IACN,CAAC;IAED,oBAAoB;QAClB,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACpD,CAAC;IAED,qBAAqB;QACnB,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACtE,IAAI,EAAE,kBAAkB;YACxB,IAAI;YACJ,EAAE;SACH,CAAC,CAAC,CAAC;IACN,CAAC;IAED,oBAAoB;QAClB,OAAO,CAAC,GAAG,IAAI,CAAC,uBAAuB,EAAE,EAAE,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,iBAAiB,CAAC,MAAmC;QACnD,OAAO,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,sBAAsB,CAAC,GAAW;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,qBAAqB,CAAC,gDAAgD,GAAG,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS;QACP,OAAO;YACL,WAAW,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE;YACpC,iBAAiB,EAAE,EAAE,GAAG,IAAI,CAAC,sBAAsB,EAAE;SACtD,CAAC;IACJ,CAAC;IAED,2BAA2B,CACzB,qBAA+B,EAC/B,MAAsB,EACtB,mBAAyC,EACzC,wBAA4C;QAE5C,OAAO,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5G,CAAC;IAED,kCAAkC,CAAC,aAAqB;QACtD,MAAM,OAAO,GAA8B,EAAE,CAAC;QAC9C,MAAM,mBAAmB,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC1D,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC,EAAE,sBAAsB,EAAE,EAAE,GAAG,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,wBAAwB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC7D,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,EAAE,2BAA2B,EAAE,EAAE,GAAG,EAAE,wBAAwB,EAAE,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,aAAa,EAAE,KAAK;gBACpB,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;aACxB,CAAC;QACJ,CAAC;QACD,OAAO;YACL,aAAa,EAAE,KAAK;YACpB,GAAG,EAAE,OAAO;SACb,CAAC;IACJ,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,OAAO,sCAAsC;IACzC,wBAAwB,GAAG,IAAI,OAAO,EAA6D,CAAC;IACpG,iCAAiC,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC/E,uBAAuB,GAAG,IAAI,OAAO,EAAwE,CAAC;IAC9G,gCAAgC,GAAG,IAAI,GAAG,EAAiC,CAAC;IAC5E,eAAe,GAAG,IAAI,GAAG,EAAyC,CAAC;IACnE,QAAQ,CAA4C;IAE5D,YAAY,WAA4C;QACtD,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5D,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACtD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAChE,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC1D,WAAW,CAAC,IAAI,CAAC,iCAAiC,EAAE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;YAClH,CAAC;YACD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,4BAA4B,EAAE,CAAC;gBAC3E,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBACzD,WAAW,CACT,IAAI,CAAC,gCAAgC,EACrC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EACxC,MAAM,CAAC,YAAY,CACpB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAwB;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,IAAI,qBAAqB,CAC7B,iBAAiB,MAAM,CAAC,UAAU,wDAAwD,CAC3F,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,sBAAsB;QACpB,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IAED,iBAAiB,CAAC,MAAmC;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,IAAI,qBAAqB,CAC7B,2BAA2B,MAAM,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,wDAAwD,CACzI,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,oBAAoB;QAClB,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED,2BAA2B,CACzB,sBAAgC,EAChC,KAAqB,EACrB,mBAAyC,EACzC,wBAA4C;QAE5C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YAC3C,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,iCAAiC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,wBAAwB,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,gCAAgC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnE,CAAC;QACD,KAAK,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1D,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IAClB,CAAC;IAED,kCAAkC,CAAC,YAAoB;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACtD,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,YAAY,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,EAA+B;IAChE,OAAO,GAAG,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,WAAW,CAAO,GAAmB,EAAE,GAAM,EAAE,KAAQ;IAC9D,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAK,CAAC;IAC9C,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpB,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,MAAM,CAAI,MAAc,EAAE,MAA+B;IAChE,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;AACH,CAAC"}

package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts

@@ -0,0 +1,6 @@
+import { Logger } from 'winston';
+import { SerializedSyncPlan, UpdateSyncRulesOptions } from '../BucketStorageFactory.js';
+/**
+ * Check if a sync config update can use incremental reprocessing.
+ */
+export declare function isCompatible(existingPlans: (SerializedSyncPlan | null)[], updateConfig: UpdateSyncRulesOptions['config'], logger: Logger): boolean;

package/dist/storage/implementation/IncrementalReprocessingCompatibility.js

@@ -0,0 +1,31 @@
+import { CompatibilityContext } from '@powersync/service-sync-rules';
+/**
+ * Check if a sync config update can use incremental reprocessing.
+ */
+export function isCompatible(existingPlans, updateConfig, logger) {
+    if (updateConfig.plan == null) {
+        // Only support sync streams with serialized plans
+        logger.info(`Not using current sync streams - incremental reprocessing not supported`);
+        return false;
+    }
+    // We don't check the storage version here - that is checked upstream.
+    if (existingPlans.length == 0) {
+        // Could technically be compatible, but there is no reason to re-use this stream.
+        return false;
+    }
+    if (existingPlans.some((plan) => plan == null)) {
+        // Only support sync streams with serialized plans
+        logger.info(`Existing replication stream not using current sync streams - incremental reprocessing not supported`);
+        return false;
+    }
+    // Technically we can compare the serialized compatibility versions? But this does not add much overhead.
+    const first = existingPlans[0];
+    const streamCompatibility = CompatibilityContext.deserialize(first.compatibility);
+    if (!streamCompatibility.equals(updateConfig.parsed.config.compatibility)) {
+        // Compatibility options must match
+        logger.info(`Compatibility options changed - incremental reprocessing not supported`);
+        return false;
+    }
+    return true;
+}
+//# sourceMappingURL=IncrementalReprocessingCompatibility.js.map

package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IncrementalReprocessingCompatibility.js","sourceRoot":"","sources":["../../../src/storage/implementation/IncrementalReprocessingCompatibility.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAIrE;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,aAA4C,EAC5C,YAA8C,EAC9C,MAAc;IAEd,IAAI,YAAY,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;QAC9B,kDAAkD;QAClD,MAAM,CAAC,IAAI,CAAC,yEAAyE,CAAC,CAAC;QACvF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sEAAsE;IACtE,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC9B,iFAAiF;QACjF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QAC/C,kDAAkD;QAClD,MAAM,CAAC,IAAI,CAAC,qGAAqG,CAAC,CAAC;QACnH,OAAO,KAAK,CAAC;IACf,CAAC;IAED,yGAAyG;IACzG,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,WAAW,CAAC,KAAM,CAAC,aAAa,CAAC,CAAC;IACnF,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAC1E,mCAAmC;QACnC,MAAM,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;QACtF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts

@@ -0,0 +1,17 @@
+import { SyncConfigWithErrors } from '@powersync/service-sync-rules';
+import { IncrementalMappingChanges, IncrementalMappingDefinitionChange, SingleSyncConfigBucketDefinitionMapping } from './BucketDefinitionMapping.js';
+export interface IncrementalDefinitionLogEntry extends IncrementalMappingDefinitionChange {
+    sourceTables?: string[];
+}
+export interface IncrementalSyncConfigUpdateLog {
+    reusedDefinitions: IncrementalDefinitionLogEntry[];
+    addedDefinitions: IncrementalDefinitionLogEntry[];
+    droppedDefinitions: IncrementalDefinitionLogEntry[];
+}
+export declare function formatIncrementalSyncConfigUpdateLog(log: IncrementalSyncConfigUpdateLog): string;
+export declare function describeIncrementalSyncConfigUpdate(options: {
+    activeMappings: SingleSyncConfigBucketDefinitionMapping[];
+    newMapping: SingleSyncConfigBucketDefinitionMapping;
+    newSyncConfig: SyncConfigWithErrors;
+    mappingChanges: IncrementalMappingChanges;
+}): IncrementalSyncConfigUpdateLog;