@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/dist/storage/BucketStorageFactory.d.ts

@@ -1,7 +1,8 @@
 import { BaseObserver } from '@powersync/lib-services-framework';
-import { SerializedCompatibilityContext, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+import { SerializedSyncPlan as RawSerializedSyncPlan, SerializedCompatibilityContext, SyncConfigWithErrors } from '@powersync/service-sync-rules';
 import { ReplicationError } from '@powersync/service-types';
-import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
+import { PersistedReplicationStream } from './PersistedReplicationStream.js';
+import { PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { ReplicationLock } from './ReplicationLock.js';
 import { ReportStorage } from './ReportStorage.js';
@@ -20,17 +21,17 @@ export declare abstract class BucketStorageFactory extends BaseObserver<BucketSt
      */
     configureSyncRules(options: UpdateSyncRulesOptions): Promise<{
         updated: boolean;
-        persisted_sync_rules?: PersistedSyncRulesContent;
+        persisted_sync_rules?: PersistedReplicationStream;
         lock?: ReplicationLock;
     }>;
     /**
      * Get a storage instance to query sync data for specific sync config.
      */
-    abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
+    abstract getInstance(replicationStream: PersistedReplicationStream, options?: GetIntanceOptions): SyncRulesBucketStorage;
     /**
      * Deploy new sync config.
      */
-    abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
+    abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedReplicationStream>;
     /**
      * Indicate that a slot was removed, and we should re-sync by creating
      * a new replication stream.
@@ -41,35 +42,23 @@ export declare abstract class BucketStorageFactory extends BaseObserver<BucketSt
      *
      * Replication should be restarted after this.
      */
-    abstract restartReplication(sync_rules_group_id: number): Promise<void>;
+    abstract restartReplication(replicationStreamId: number): Promise<void>;
     /**
-     * Get the sync config used for querying.
+     * Get the sync config and storage used for querying.
      */
-    getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
+    abstract getActiveSyncConfig(): Promise<ResolvedSyncConfig | null>;
     /**
-     * Get the sync config used for querying.
+     * Get the sync config and storage that is still deploying.
      */
-    abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
+    abstract getDeployingSyncConfig(): Promise<ResolvedSyncConfig | null>;
     /**
-     * Get the sync config that will be active next once done with initial replicatino.
+     * Get all replication streams currently replicating.
      */
-    getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
+    abstract getReplicatingReplicationStreams(): Promise<PersistedReplicationStream[]>;
     /**
-     * Get the sync config that will be active next once done with initial replicatino.
+     * Get all replication streams stopped but not terminated yet.
      */
-    abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
-    /**
-     * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
-     */
-    abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
-    /**
-     * Get all sync config stopped but not terminated yet.
-     */
-    abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
-    /**
-     * Get the active storage instance.
-     */
-    abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
+    abstract getStoppedReplicationStreams(): Promise<PersistedReplicationStream[]>;
     /**
      * Get storage size of active replication stream.
      */
@@ -88,6 +77,11 @@ export interface BucketStorageFactoryListener {
     syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
     replicationEvent: (event: ReplicationEventPayload) => void;
 }
+export interface ResolvedSyncConfig {
+    content: PersistedSyncConfigContent;
+    replicationStream: PersistedReplicationStream;
+    storage: SyncRulesBucketStorage;
+}
 export interface StorageMetrics {
     /**
      * Size of operations (bucket_data)
@@ -103,8 +97,28 @@ export interface StorageMetrics {
      * Size of current_data.
      */
     replication_size_bytes: number;
+    /**
+     * Size of active object-storage references, such as S3 bucket-data objects.
+     */
+    object_storage_size_bytes?: number;
+    /**
+     * Per-sync-config storage sizes, when the storage backend can provide them.
+     */
+    sync_config_metrics?: StorageSyncConfigMetrics[];
+}
+export interface StorageSyncConfigMetrics {
+    sync_config_id: string;
+    sync_config_state: string;
+    /** Optional operator-supplied label identifying this sync config version. */
+    version_label?: string;
+    attributed_bucket_data_bytes: number;
+    attributed_parameter_indexes_bytes: number;
+    attributed_source_records_bytes: number;
+    attributed_object_storage_bytes: number;
 }
 export interface UpdateSyncRulesOptions {
+    /** Optional operator-supplied label identifying this deployed sync config version. */
+    version_label?: string;
     config: {
         yaml: string;
         /**
@@ -120,6 +134,11 @@ export interface UpdateSyncRulesOptions {
     };
     lock?: boolean;
     storageVersion?: number;
+    /**
+     * Start a replacement replication stream instead of incrementally reprocessing a
+     * compatible active stream. Used when the current stream needs a full reprocess.
+     */
+    forceNewReplicationStream?: boolean;
     /**
      * Only relevant if the result is used. This does not affect the persisted config.
      */
@@ -129,14 +148,13 @@ export interface SerializedSyncPlan {
     /**
      * The serialized plan, from {@link serializeSyncPlan}.
      */
-    plan: unknown;
+    plan: RawSerializedSyncPlan;
     compatibility: SerializedCompatibilityContext;
     /**
-     * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
-     * yet.
+     * Raw event SQL persisted as a compatibility mirror for compiled {@link plan} events.
      *
-     * We might revisit that in the future, but for now we store SQL text of their definitions here to be able to restore
-     * them.
+     * Compiled events are an additive plan field. Older services ignore that field and restore these descriptors through
+     * the legacy evaluator. Keep dual-writing this field until a future plan version explicitly removes that support.
      */
     eventDescriptors: Record<string, string[]>;
     errors?: ReplicationError[];
@@ -188,4 +206,5 @@ export interface TestStorageConfig {
     factory: TestStorageFactory;
     tableIdStrings: boolean;
     storageVersion?: number;
+    compressedBucketStorage?: boolean;
 }

package/dist/storage/BucketStorageFactory.js

@@ -14,13 +14,16 @@ export class BucketStorageFactory extends BaseObserver {
      * Update sync config from configuration, if changed.
      */
     async configureSyncRules(options) {
-        const next = await this.getNextSyncRulesContent();
-        const active = await this.getActiveSyncRulesContent();
-        if (next?.sync_rules_content == options.config.yaml) {
+        const deploying = await this.getDeployingSyncConfig();
+        const active = await this.getActiveSyncConfig();
+        if (deploying?.content.sync_rules_content == options.config.yaml &&
+            deploying.content.version_label == options.version_label) {
             logger.info('Sync config unchanged');
             return { updated: false };
         }
-        else if (next == null && active?.sync_rules_content == options.config.yaml) {
+        else if (deploying == null &&
+            active?.content.sync_rules_content == options.config.yaml &&
+            active.content.version_label == options.version_label) {
             logger.info('Sync config unchanged');
             return { updated: false };
         }
@@ -30,20 +33,6 @@ export class BucketStorageFactory extends BaseObserver {
             return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
         }
     }
-    /**
-     * Get the sync config used for querying.
-     */
-    async getActiveSyncRules(options) {
-        const content = await this.getActiveSyncRulesContent();
-        return content?.parsed(options) ?? null;
-    }
-    /**
-     * Get the sync config that will be active next once done with initial replicatino.
-     */
-    async getNextSyncRules(options) {
-        const content = await this.getNextSyncRulesContent();
-        return content?.parsed(options) ?? null;
-    }
 }
 export function updateSyncRulesFromYaml(content, options) {
     const config = SqlSyncRules.fromYaml(content, {
@@ -59,12 +48,13 @@ export function updateSyncRulesFromConfig(parsed, options) {
     const { config, errors } = parsed;
     if (config instanceof PrecompiledSyncConfig) {
         const eventDescriptors = {};
-        for (const event of config.eventDescriptors) {
-            eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
+        for (const event of config.plan.events) {
+            eventDescriptors[event.name] = event.sourceQueries.map((query) => query.sql);
         }
         plan = {
             compatibility: config.compatibility.serialize(),
             plan: serializeSyncPlan(config.plan),
+            // Dual-write raw SQL so older services can ignore additive compiled plan events without losing event behavior.
             eventDescriptors,
             errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
         };

package/dist/storage/BucketStorageFactory.js.map

@@ -1 +1 @@
-{"version":3,"file":"BucketStorageFactory.js","sourceRoot":"","sources":["../../src/storage/BucketStorageFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EACL,qBAAqB,EAErB,iBAAiB,EACjB,YAAY,EAEb,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AAO1E;;;;;;;GAOG;AACH,MAAM,OAAgB,oBACpB,SAAQ,YAA0C;IAGlD;;OAEG;IACH,KAAK,CAAC,kBAAkB,CACtB,OAA+B;QAE/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnC,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAwBD;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAA8B;QACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAOD;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,OAA8B;QACnD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACrD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;CAsCF;AAmED,MAAM,UAAU,uBAAuB,CACrC,OAAe,EACf,OAAyE;IAEzE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE;QAC5C,2CAA2C;QAC3C,MAAM,EAAE,SAAS;QACjB,aAAa,EAAE,OAAO,EAAE,aAAa,IAAI,gBAAgB,EAAE,4BAA4B;QACvF,YAAY,EAAE,OAAO,EAAE,QAAQ,IAAI,KAAK;KACzC,CAAC,CAAC;IAEH,OAAO,yBAAyB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,MAA4B,EAC5B,OAAgD;IAEhD,IAAI,IAAI,GAA8B,IAAI,CAAC;IAC3C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAClC,IAAI,MAAM,YAAY,qBAAqB,EAAE,CAAC;QAC5C,MAAM,gBAAgB,GAA6B,EAAE,CAAC;QACtD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5C,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,GAAG;YACL,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE;YAC/C,IAAI,EAAE,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC;YACpC,gBAAgB;YAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC;SACpE,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC;AACxE,CAAC"}
+{"version":3,"file":"BucketStorageFactory.js","sourceRoot":"","sources":["../../src/storage/BucketStorageFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EACL,qBAAqB,EAGrB,iBAAiB,EACjB,YAAY,EAEb,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AAQ1E;;;;;;;GAOG;AACH,MAAM,OAAgB,oBACpB,SAAQ,YAA0C;IAGlD;;OAEG;IACH,KAAK,CAAC,kBAAkB,CACtB,OAA+B;QAE/B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAEhD,IACE,SAAS,EAAE,OAAO,CAAC,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI;YAC5D,SAAS,CAAC,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,aAAa,EACxD,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IACL,SAAS,IAAI,IAAI;YACjB,MAAM,EAAE,OAAO,CAAC,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI;YACzD,MAAM,CAAC,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,aAAa,EACrD,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnC,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;CA+DF;AAqGD,MAAM,UAAU,uBAAuB,CACrC,OAAe,EACf,OAAyE;IAEzE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE;QAC5C,2CAA2C;QAC3C,MAAM,EAAE,SAAS;QACjB,aAAa,EAAE,OAAO,EAAE,aAAa,IAAI,gBAAgB,EAAE,4BAA4B;QACvF,YAAY,EAAE,OAAO,EAAE,QAAQ,IAAI,KAAK;KACzC,CAAC,CAAC;IAEH,OAAO,yBAAyB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,MAA4B,EAC5B,OAAgD;IAEhD,IAAI,IAAI,GAA8B,IAAI,CAAC;IAC3C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAClC,IAAI,MAAM,YAAY,qBAAqB,EAAE,CAAC;QAC5C,MAAM,gBAAgB,GAA6B,EAAE,CAAC;QACtD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACvC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,GAAG;YACL,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE;YAC/C,IAAI,EAAE,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC;YACpC,+GAA+G;YAC/G,gBAAgB;YAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC;SACpE,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC;AACxE,CAAC"}

package/dist/storage/CheckpointChecksumInvalidatedError.d.ts

@@ -0,0 +1,12 @@
+import { InternalOpId } from '../util/util-index.js';
+/**
+ * A checkpoint cannot be served because compaction rewrote a bucket-data
+ * document across its end boundary.
+ *
+ * The sync loop must skip this checkpoint before it sends its checkpoint line.
+ */
+export declare class CheckpointChecksumInvalidatedError extends Error {
+    readonly checkpoint: InternalOpId;
+    readonly bucket: string;
+    constructor(checkpoint: InternalOpId, bucket: string);
+}

package/dist/storage/CheckpointChecksumInvalidatedError.js

@@ -0,0 +1,17 @@
+/**
+ * A checkpoint cannot be served because compaction rewrote a bucket-data
+ * document across its end boundary.
+ *
+ * The sync loop must skip this checkpoint before it sends its checkpoint line.
+ */
+export class CheckpointChecksumInvalidatedError extends Error {
+    checkpoint;
+    bucket;
+    constructor(checkpoint, bucket) {
+        super(`Checkpoint ${checkpoint} was invalidated by compaction in bucket ${bucket}`);
+        this.checkpoint = checkpoint;
+        this.bucket = bucket;
+        this.name = 'CheckpointChecksumInvalidatedError';
+    }
+}
+//# sourceMappingURL=CheckpointChecksumInvalidatedError.js.map

package/dist/storage/CheckpointChecksumInvalidatedError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CheckpointChecksumInvalidatedError.js","sourceRoot":"","sources":["../../src/storage/CheckpointChecksumInvalidatedError.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,OAAO,kCAAmC,SAAQ,KAAK;IAEzC;IACA;IAFlB,YACkB,UAAwB,EACxB,MAAc;QAE9B,KAAK,CAAC,cAAc,UAAU,4CAA4C,MAAM,EAAE,CAAC,CAAC;QAHpE,eAAU,GAAV,UAAU,CAAc;QACxB,WAAM,GAAN,MAAM,CAAQ;QAG9B,IAAI,CAAC,IAAI,GAAG,oCAAoC,CAAC;IACnD,CAAC;CACF"}

package/dist/storage/ChecksumCache.d.ts

@@ -10,19 +10,24 @@ export interface FetchPartialBucketChecksum {
 }
 export type PartialOrFullChecksum = PartialChecksum | BucketChecksum;
 export type PartialChecksumMap = Map<string, PartialOrFullChecksum>;
-export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<PartialChecksumMap>;
-export interface ChecksumCacheOptions {
+export type FetchChecksums<TContext = undefined> = (batch: FetchPartialBucketChecksum[], context: TContext) => Promise<PartialChecksumMap>;
+export interface ChecksumCacheOptions<TContext = undefined> {
     /**
      * Upstream checksum implementation.
      *
      * This fetches a batch of either entire bucket checksums, or a partial range.
      */
-    fetchChecksums: FetchChecksums;
+    fetchChecksums: FetchChecksums<TContext>;
     /**
      * Maximum number of cached checksums.
      */
     maxSize?: number;
+    /**
+     * How long to keep cached checksums before refreshing them.
+     */
+    ttlMs?: number;
 }
+export declare const DEFAULT_CHECKSUM_CACHE_TTL_MS: number;
 /**
  * Implement a LRU cache for checksum requests. Each (bucket, checkpoint) request is cached separately,
  * while the lookups occur in batches.
@@ -32,7 +37,7 @@ export interface ChecksumCacheOptions {
  *
  * We use the LRUCache fetchMethod to deduplicate in-progress requests.
  */
-export declare class ChecksumCache {
+export declare class ChecksumCache<TContext = undefined> {
     /**
      * The primary checksum cache, with key of `${checkpoint}/${bucket}`.
      */
@@ -42,7 +47,7 @@ export declare class ChecksumCache {
      */
     private bucketCheckpoints;
     private fetchChecksums;
-    constructor(options: ChecksumCacheOptions);
+    constructor(options: ChecksumCacheOptions<TContext>);
     clear(): void;
     getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<BucketChecksum[]>;
     /**
@@ -52,5 +57,5 @@ export declare class ChecksumCache {
      *
      * @returns a Map with exactly one entry for each bucket requested
      */
-    getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap>;
+    getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[], fetchContext?: TContext): Promise<ChecksumMap>;
 }

package/dist/storage/ChecksumCache.js

@@ -3,7 +3,8 @@ import { LRUCache } from 'lru-cache/min';
 import { addBucketChecksums } from '../util/utils.js';
 // Approximately 5MB of memory, if we assume 50 bytes per entry
 const DEFAULT_MAX_SIZE = 100_000;
-const TTL_MS = 3_600_000;
+// Default to 1h
+export const DEFAULT_CHECKSUM_CACHE_TTL_MS = 1 * 3_600_000;
 /**
  * Implement a LRU cache for checksum requests. Each (bucket, checkpoint) request is cached separately,
  * while the lookups occur in batches.
@@ -60,7 +61,7 @@ export class ChecksumCache {
             // We use a TTL so that counts can eventually be refreshed
             // after a compact. This only has effect if the bucket has
             // not been checked in the meantime.
-            ttl: TTL_MS,
+            ttl: options.ttlMs ?? DEFAULT_CHECKSUM_CACHE_TTL_MS,
             ttlResolution: 1_000,
             allowStale: false
         });
@@ -81,7 +82,7 @@ export class ChecksumCache {
      *
      * @returns a Map with exactly one entry for each bucket requested
      */
-    async getChecksumMap(checkpoint, buckets) {
+    async getChecksumMap(checkpoint, buckets, fetchContext) {
         // Buckets that don't have a cached checksum for this checkpoint yet
         let toFetch = new Set();
         // Newly fetched results
@@ -201,7 +202,7 @@ export class ChecksumCache {
                     bucketRequests.push(bucketRequest);
                 }
                 // Fetch partial checksums from upstream
-                const results = await this.fetchChecksums(bucketRequests);
+                const results = await this.fetchChecksums(bucketRequests, fetchContext);
                 for (let bucket of toFetch) {
                     const result = results.get(bucket);
                     const toAdd = add.get(bucket);

package/dist/storage/ChecksumCache.js.map

@@ -1 +1 @@
-{"version":3,"file":"ChecksumCache.js","sourceRoot":"","sources":["../../src/storage/ChecksumCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,kBAAkB,EAA8C,MAAM,kBAAkB,CAAC;AAkClG,+DAA+D;AAC/D,MAAM,gBAAgB,GAAG,OAAO,CAAC;AAEjC,MAAM,MAAM,GAAG,SAAS,CAAC;AAEzB;;;;;;;;GAQG;AACH,MAAM,OAAO,aAAa;IACxB;;OAEG;IACK,KAAK,CAAyD;IACtE;;OAEG;IACK,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAE1D,cAAc,CAAiB;IAEvC,YAAY,OAA6B;QACvC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAA+C;YACtE,GAAG,EAAE,OAAO,CAAC,OAAO,IAAI,gBAAgB;YACxC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE;gBACpD,oDAAoD;gBACpD,sEAAsE;gBACtE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAEnD,uDAAuD;gBACvD,IAAI,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACvD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,aAAa,GAAG,IAAI,UAAU,EAAE,CAAC;oBACjC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;gBACpD,CAAC;gBACD,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACjD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACtB,2DAA2D;gBAC3D,MAAM,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;gBAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBACD,aAAa,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;gBAC5C,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,cAAc,EAAE,IAAI;YAEpB,8EAA8E;YAC9E,oCAAoC;YACpC,gBAAgB,EAAE,IAAI;YAEtB,0DAA0D;YAC1D,0DAA0D;YAC1D,oCAAoC;YACpC,GAAG,EAAE,MAAM;YACX,aAAa,EAAE,KAAK;YACpB,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAgC;QAC3E,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACjE,kDAAkD;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC,CAAC;IAChE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,UAAwB,EAAE,OAAgC;QAC7E,oEAAoE;QACpE,IAAI,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAEhC,wBAAwB;QACxB,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,4CAA4C;QAC5C,IAAI,YAAyB,CAAC;QAC9B,IAAI,WAAgC,CAAC;QACrC,IAAI,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,YAAY,GAAG,OAAO,CAAC;YACvB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,MAAM,OAAO,GAAyB;YACpC,KAAK,CAAC,KAAK,CAAC,MAAM;gBAChB,MAAM,YAAY,CAAC;gBACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;gBACD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAC;gBACnE,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;SAC/B,CAAC;QAEF,+DAA+D;QAC/D,IAAI,cAAc,GAAiD,IAAI,CAAC;QAExE,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4B,CAAC;QAEtD,IAAI,CAAC;YACH,kCAAkC;YAClC,IAAI,kBAAkB,GAAoB,EAAE,CAAC;YAE7C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACzD,IAAI,MAAM,GAAoC,EAAE,CAAC;gBACjD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;oBAC5F,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;wBACtB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;oBAC7D,CAAC;oBACD,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC7C,CAAC,CAAC,CAAC;gBACH,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,IAAI,MAAM,CAAC,KAAK,IAAI,KAAK,IAAI,MAAM,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxD,2EAA2E;oBAC3E,uBAAuB;oBACvB,yDAAyD;gBAC3D,CAAC;qBAAM,CAAC;oBACN,2CAA2C;oBAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAC3B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YACD,6DAA6D;YAC7D,wDAAwD;YACxD,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAExD,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;gBACtB,wCAAwC;gBACxC,YAAY,EAAE,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,IAAI,cAAc,GAAiC,EAAE,CAAC;gBACtD,mEAAmE;gBACnE,IAAI,GAAG,GAAG,IAAI,GAAG,EAA0B,CAAC;gBAE5C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,aAAa,GAAsC,IAAI,CAAC;oBAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACzD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,uDAAuD;wBACvD,IAAI,IAAI,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;wBAC/D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC;wBACpC,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;4BAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;4BACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;4BAC1C,mCAAmC;4BACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;4BACzC,iEAAiE;4BACjE,uEAAuE;4BACvE,8DAA8D;4BAC9D,8CAA8C;4BAC9C,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gCACnB,2DAA2D;gCAC3D,aAAa,GAAG;oCACd,MAAM;oCACN,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE;oCAC9B,KAAK,EAAE,EAAE;oCACT,GAAG,EAAE,UAAU;iCAChB,CAAC;gCACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gCACxB,MAAM;4BACR,CAAC;4BAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gCACvB,yBAAyB;gCACzB,MAAM;4BACR,CAAC;4BACD,oBAAoB;4BACpB,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACpB,CAAC;oBACH,CAAC;oBAED,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,+DAA+D;wBAC/D,aAAa,GAAG;4BACd,MAAM;4BACN,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE;4BAC9B,GAAG,EAAE,UAAU;yBAChB,CAAC;wBACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;4BACd,MAAM;4BACN,QAAQ,EAAE,CAAC;4BACX,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBACrC,CAAC;gBAED,wCAAwC;gBACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;gBAE1D,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;wBAClB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,EAAE,CAAC,CAAC;oBAC9C,CAAC;oBACD,mDAAmD;oBACnD,6DAA6D;oBAC7D,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;oBACxD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBAClC,CAAC;gBAED,6DAA6D;gBAC7D,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,wDAAwD;YACxD,iFAAiF;YACjF,uBAAuB;YACvB,WAAW,CAAC,CAAC,CAAC,CAAC;YAEf,yFAAyF;YACzF,6BAA6B;YAC7B,MAAM,cAAc,CAAC;YAErB,MAAM,CAAC,CAAC;QACV,CAAC;QAED,gDAAgD;QAChD,MAAM,cAAc,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACpD,8BAA8B;QAC9B,KAAK,IAAI,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACxC,oBAAoB;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,YAAY,CAAC,IAAI,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;CACF;AAED,SAAS,YAAY,CAAC,UAAiC,EAAE,MAAc;IACrE,OAAO,GAAG,UAAU,IAAI,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;AAC3F,CAAC"}
+{"version":3,"file":"ChecksumCache.js","sourceRoot":"","sources":["../../src/storage/ChecksumCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,kBAAkB,EAA8C,MAAM,kBAAkB,CAAC;AA0ClG,+DAA+D;AAC/D,MAAM,gBAAgB,GAAG,OAAO,CAAC;AAEjC,gBAAgB;AAChB,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,GAAG,SAAS,CAAC;AAE3D;;;;;;;;GAQG;AACH,MAAM,OAAO,aAAa;IACxB;;OAEG;IACK,KAAK,CAAmE;IAChF;;OAEG;IACK,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAE1D,cAAc,CAA2B;IAEjD,YAAY,OAAuC;QACjD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAyD;YAChF,GAAG,EAAE,OAAO,CAAC,OAAO,IAAI,gBAAgB;YACxC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE;gBACpD,oDAAoD;gBACpD,sEAAsE;gBACtE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAEnD,uDAAuD;gBACvD,IAAI,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACvD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,aAAa,GAAG,IAAI,UAAU,EAAE,CAAC;oBACjC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;gBACpD,CAAC;gBACD,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACjD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACtB,2DAA2D;gBAC3D,MAAM,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;gBAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBACD,aAAa,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;gBAC5C,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,cAAc,EAAE,IAAI;YAEpB,8EAA8E;YAC9E,oCAAoC;YACpC,gBAAgB,EAAE,IAAI;YAEtB,0DAA0D;YAC1D,0DAA0D;YAC1D,oCAAoC;YACpC,GAAG,EAAE,OAAO,CAAC,KAAK,IAAI,6BAA6B;YACnD,aAAa,EAAE,KAAK;YACpB,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAgC;QAC3E,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACjE,kDAAkD;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC,CAAC;IAChE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAClB,UAAwB,EACxB,OAAgC,EAChC,YAAuB;QAEvB,oEAAoE;QACpE,IAAI,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAEhC,wBAAwB;QACxB,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,4CAA4C;QAC5C,IAAI,YAAyB,CAAC;QAC9B,IAAI,WAAgC,CAAC;QACrC,IAAI,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,YAAY,GAAG,OAAO,CAAC;YACvB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,MAAM,OAAO,GAAmC;YAC9C,KAAK,CAAC,KAAK,CAAC,MAAM;gBAChB,MAAM,YAAY,CAAC;gBACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;gBACD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAC;gBACnE,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;SAC/B,CAAC;QAEF,+DAA+D;QAC/D,IAAI,cAAc,GAAiD,IAAI,CAAC;QAExE,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4B,CAAC;QAEtD,IAAI,CAAC;YACH,kCAAkC;YAClC,IAAI,kBAAkB,GAAoB,EAAE,CAAC;YAE7C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACzD,IAAI,MAAM,GAAoC,EAAE,CAAC;gBACjD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;oBAC5F,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;wBACtB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;oBAC7D,CAAC;oBACD,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC7C,CAAC,CAAC,CAAC;gBACH,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,IAAI,MAAM,CAAC,KAAK,IAAI,KAAK,IAAI,MAAM,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxD,2EAA2E;oBAC3E,uBAAuB;oBACvB,yDAAyD;gBAC3D,CAAC;qBAAM,CAAC;oBACN,2CAA2C;oBAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAC3B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YACD,6DAA6D;YAC7D,wDAAwD;YACxD,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAExD,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;gBACtB,wCAAwC;gBACxC,YAAY,EAAE,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,IAAI,cAAc,GAAiC,EAAE,CAAC;gBACtD,mEAAmE;gBACnE,IAAI,GAAG,GAAG,IAAI,GAAG,EAA0B,CAAC;gBAE5C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,aAAa,GAAsC,IAAI,CAAC;oBAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACzD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,uDAAuD;wBACvD,IAAI,IAAI,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;wBAC/D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC;wBACpC,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;4BAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;4BACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;4BAC1C,mCAAmC;4BACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;4BACzC,iEAAiE;4BACjE,uEAAuE;4BACvE,8DAA8D;4BAC9D,8CAA8C;4BAC9C,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gCACnB,2DAA2D;gCAC3D,aAAa,GAAG;oCACd,MAAM;oCACN,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE;oCAC9B,KAAK,EAAE,EAAE;oCACT,GAAG,EAAE,UAAU;iCAChB,CAAC;gCACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gCACxB,MAAM;4BACR,CAAC;4BAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gCACvB,yBAAyB;gCACzB,MAAM;4BACR,CAAC;4BACD,oBAAoB;4BACpB,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACpB,CAAC;oBACH,CAAC;oBAED,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,+DAA+D;wBAC/D,aAAa,GAAG;4BACd,MAAM;4BACN,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE;4BAC9B,GAAG,EAAE,UAAU;yBAChB,CAAC;wBACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;4BACd,MAAM;4BACN,QAAQ,EAAE,CAAC;4BACX,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBACrC,CAAC;gBAED,wCAAwC;gBACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,YAAwB,CAAC,CAAC;gBAEpF,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;wBAClB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,EAAE,CAAC,CAAC;oBAC9C,CAAC;oBACD,mDAAmD;oBACnD,6DAA6D;oBAC7D,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;oBACxD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBAClC,CAAC;gBAED,6DAA6D;gBAC7D,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,wDAAwD;YACxD,iFAAiF;YACjF,uBAAuB;YACvB,WAAW,CAAC,CAAC,CAAC,CAAC;YAEf,yFAAyF;YACzF,6BAA6B;YAC7B,MAAM,cAAc,CAAC;YAErB,MAAM,CAAC,CAAC;QACV,CAAC;QAED,gDAAgD;QAChD,MAAM,cAAc,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACpD,8BAA8B;QAC9B,KAAK,IAAI,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACxC,oBAAoB;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,YAAY,CAAC,IAAI,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;CACF;AAED,SAAS,YAAY,CAAC,UAAiC,EAAE,MAAc;IACrE,OAAO,GAAG,UAAU,IAAI,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;AAC3F,CAAC"}

package/dist/storage/ParsedSyncConfigSet.d.ts

@@ -0,0 +1,23 @@
+import { HydratedSyncConfig, HydrationState, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+/**
+ * A parsed sync config set is the identity boundary for parsed sync config state.
+ *
+ * All values on a single instance - the parsed configs, hydration state and hydrated
+ * sync config - are derived from the same parse, and may be associated with each other
+ * by object identity. Values from two different instances must never be mixed, even if
+ * they were parsed from the same persisted content.
+ */
+export interface ParsedSyncConfigSet {
+    readonly replicationStreamId: number;
+    readonly replicationStreamName: string;
+    readonly syncConfigs: SyncConfigWithErrors[];
+    /**
+     * For testing only.
+     */
+    readonly hydrationState: HydrationState;
+    /**
+     * The hydrated sync config for this parse. This is a stable property: repeated reads
+     * return the same instance.
+     */
+    readonly hydratedSyncConfig: HydratedSyncConfig;
+}

package/dist/storage/ParsedSyncConfigSet.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=ParsedSyncConfigSet.js.map

package/dist/storage/ParsedSyncConfigSet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ParsedSyncConfigSet.js","sourceRoot":"","sources":["../../src/storage/ParsedSyncConfigSet.ts"],"names":[],"mappings":""}

package/dist/storage/PersistedReplicationStream.d.ts

@@ -0,0 +1,32 @@
+import { Logger } from 'winston';
+import { SyncRuleState } from './BucketStorage.js';
+import type { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
+import type { ParseSyncConfigOptions, PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
+import { ReplicationLock } from './ReplicationLock.js';
+import { StorageVersionConfig } from './StorageVersionConfig.js';
+export declare abstract class PersistedReplicationStream implements PersistedReplicationStreamData {
+    readonly replicationStreamId: number;
+    readonly replicationJobId: string;
+    readonly replicationStreamName: string;
+    readonly state: SyncRuleState;
+    readonly storageVersion: number;
+    readonly logger: Logger;
+    abstract readonly syncConfigContent: readonly PersistedSyncConfigContent[];
+    abstract readonly current_lock: ReplicationLock | null;
+    constructor(data: PersistedReplicationStreamData);
+    getStorageConfig(): StorageVersionConfig;
+    abstract lock(): Promise<ReplicationLock>;
+    abstract parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
+}
+export interface PersistedReplicationStreamData {
+    readonly replicationStreamId: number;
+    readonly replicationStreamName: string;
+    readonly state: SyncRuleState;
+    readonly storageVersion: number;
+    /**
+     * Uniquely identifies a job, as a combination of replication stream and processing sync configs.
+     *
+     * Used in-memory only.
+     */
+    readonly replicationJobId?: string;
+}

package/dist/storage/PersistedReplicationStream.js

@@ -0,0 +1,26 @@
+import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { STORAGE_VERSION_CONFIG } from './StorageVersionConfig.js';
+export class PersistedReplicationStream {
+    replicationStreamId;
+    replicationJobId;
+    replicationStreamName;
+    state;
+    storageVersion;
+    logger;
+    constructor(data) {
+        this.replicationStreamId = data.replicationStreamId;
+        this.replicationJobId = data.replicationJobId ?? String(data.replicationStreamId);
+        this.replicationStreamName = data.replicationStreamName;
+        this.state = data.state;
+        this.storageVersion = data.storageVersion;
+        this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
+    }
+    getStorageConfig() {
+        const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
+        if (storageConfig == null) {
+            throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`);
+        }
+        return storageConfig;
+    }
+}
+//# sourceMappingURL=PersistedReplicationStream.js.map

package/dist/storage/PersistedReplicationStream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PersistedReplicationStream.js","sourceRoot":"","sources":["../../src/storage/PersistedReplicationStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAMrG,OAAO,EAAE,sBAAsB,EAAwB,MAAM,2BAA2B,CAAC;AAEzF,MAAM,OAAgB,0BAA0B;IACrC,mBAAmB,CAAS;IAC5B,gBAAgB,CAAS;IACzB,qBAAqB,CAAS;IAC9B,KAAK,CAAgB;IACrB,cAAc,CAAS;IACvB,MAAM,CAAS;IAKxB,YAAY,IAAoC;QAC9C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACpD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAClF,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC;QACxD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,gBAAgB;QACd,MAAM,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClE,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,+BAA+B,IAAI,CAAC,cAAc,2BAA2B,IAAI,CAAC,mBAAmB,EAAE,CACxG,CAAC;QACJ,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;CAKF"}

package/dist/storage/PersistedSyncConfigContent.d.ts

@@ -0,0 +1,65 @@
+import { SyncConfigWithErrors } from '@powersync/service-sync-rules';
+import { Logger } from 'winston';
+import { SyncRuleState } from './BucketStorage.js';
+import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
+import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
+import { PersistedSyncConfigStatus } from './PersistedSyncConfigStatus.js';
+import { StorageVersionConfig } from './StorageVersionConfig.js';
+export interface ParsePersistedSyncConfigContentOptions {
+    content: string;
+    compiledPlan: SerializedSyncPlan | null;
+    storageVersion: number;
+    parseOptions: ParseSyncConfigOptions;
+}
+export declare function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfigContentOptions): SyncConfigWithErrors;
+/**
+ * Immutable sync config content for one sync config inside a replication stream.
+ *
+ * This represents the persisted config content. Fetch per-config status with
+ * {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
+ * lifecycle concerns such as locking and termination.
+ */
+export declare abstract class PersistedSyncConfigContent implements PersistedSyncConfigContentData {
+    readonly replicationStreamId: number;
+    readonly sync_rules_content: string;
+    readonly compiled_plan: SerializedSyncPlan | null;
+    readonly replicationStreamName: string;
+    readonly storageVersion: number;
+    readonly logger: Logger;
+    readonly syncConfigId: PersistedSyncConfigId | null;
+    readonly syncConfigState: SyncRuleState;
+    readonly version_label: string | undefined;
+    constructor(data: PersistedSyncConfigContentData);
+    /**
+     * Load the storage config.
+     *
+     * This may throw if the persisted storage version is not supported.
+     */
+    getStorageConfig(): StorageVersionConfig;
+    /**
+     * Parse only this config's content into a single {@link SyncConfigWithErrors}.
+     *
+     * This does not depend on any other configs in the same replication stream.
+     */
+    protected parseSingleConfig(options: ParseSyncConfigOptions): SyncConfigWithErrors;
+    parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
+    asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions;
+    /**
+     * Fetch the current persisted state for this exact sync config.
+     */
+    abstract getSyncConfigStatus(): Promise<PersistedSyncConfigStatus | null>;
+}
+export interface PersistedSyncConfigContentData {
+    readonly replicationStreamId: number;
+    readonly sync_rules_content: string;
+    readonly compiled_plan: SerializedSyncPlan | null;
+    readonly replicationStreamName: string;
+    readonly storageVersion: number;
+    readonly syncConfigId?: PersistedSyncConfigId | null;
+    readonly syncConfigState: SyncRuleState;
+    readonly version_label?: string;
+}
+export type PersistedSyncConfigId = string;
+export interface ParseSyncConfigOptions {
+    defaultSchema: string;
+}