@powersync/service-core 0.0.0-dev-20250214100224 → 0.0.0-dev-20250227082606

package/dist/storage/storage-index.d.ts

@@ -7,3 +7,8 @@ export * from './SourceTable.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';
 export * from './WriteCheckpointAPI.js';
+export * from './BucketStorageFactory.js';
+export * from './BucketStorageBatch.js';
+export * from './SyncRulesBucketStorage.js';
+export * from './PersistedSyncRulesContent.js';
+export * from './ReplicationLock.js';

package/dist/storage/storage-index.js

@@ -7,4 +7,9 @@ export * from './SourceTable.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';
 export * from './WriteCheckpointAPI.js';
+export * from './BucketStorageFactory.js';
+export * from './BucketStorageBatch.js';
+export * from './SyncRulesBucketStorage.js';
+export * from './PersistedSyncRulesContent.js';
+export * from './ReplicationLock.js';
 //# sourceMappingURL=storage-index.js.map

package/dist/storage/storage-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mBAAmB,CAAC;AAClC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC"}
+{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mBAAmB,CAAC;AAClC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,yBAAyB,CAAC;AACxC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,sBAAsB,CAAC"}

package/dist/sync/BucketChecksumState.d.ts

@@ -0,0 +1,95 @@
+import { BucketDescription, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
+import { BucketSyncState } from './sync.js';
+import { SyncContext } from './SyncContext.js';
+export interface BucketChecksumStateOptions {
+    syncContext: SyncContext;
+    bucketStorage: BucketChecksumStateStorage;
+    syncRules: SqlSyncRules;
+    syncParams: RequestParameters;
+    initialBucketPositions?: {
+        name: string;
+        after: string;
+    }[];
+}
+/**
+ * Represents the state of the checksums and data for a specific connection.
+ *
+ * Handles incrementally re-computing checkpoints.
+ */
+export declare class BucketChecksumState {
+    private readonly context;
+    private readonly bucketStorage;
+    /**
+     * Bucket state of bucket id -> op_id.
+     * This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
+     */
+    bucketDataPositions: Map<string, BucketSyncState>;
+    /**
+     * Last checksums sent to the client. We keep this to calculate checkpoint diffs.
+     */
+    private lastChecksums;
+    private lastWriteCheckpoint;
+    private readonly parameterState;
+    /**
+     * Keep track of buckets that need to be downloaded. This is specifically relevant when
+     * partial checkpoints are sent.
+     */
+    private pendingBucketDownloads;
+    constructor(options: BucketChecksumStateOptions);
+    buildNextCheckpointLine(next: storage.StorageCheckpointUpdate): Promise<CheckpointLine | null>;
+    /**
+     * Get bucket positions to sync, given the list of buckets.
+     *
+     * @param bucketsToFetch List of buckets to fetch, typically from buildNextCheckpointLine, or a subset of that
+     * @returns
+     */
+    getFilteredBucketPositions(bucketsToFetch: BucketDescription[]): Map<string, string>;
+    /**
+     * Update the position of bucket data the client has.
+     *
+     * @param bucket the bucket name
+     * @param nextAfter sync operations >= this value in the next batch
+     */
+    updateBucketPosition(options: {
+        bucket: string;
+        nextAfter: string;
+        hasMore: boolean;
+    }): void;
+}
+export interface CheckpointUpdate {
+    /**
+     * All buckets forming part of the checkpoint.
+     */
+    buckets: BucketDescription[];
+    /**
+     * If present, a set of buckets that have been updated since the last checkpoint.
+     *
+     * If null, assume that any bucket in `buckets` may have been updated.
+     */
+    updatedBuckets: Set<string> | null;
+}
+export declare class BucketParameterState {
+    private readonly context;
+    readonly bucketStorage: BucketChecksumStateStorage;
+    readonly syncRules: SqlSyncRules;
+    readonly syncParams: RequestParameters;
+    private readonly querier;
+    private readonly staticBuckets;
+    constructor(context: SyncContext, bucketStorage: BucketChecksumStateStorage, syncRules: SqlSyncRules, syncParams: RequestParameters);
+    getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate | null>;
+    /**
+     * For static buckets, we can keep track of which buckets have been updated.
+     */
+    private getCheckpointUpdateStatic;
+    /**
+     * For dynamic buckets, we need to re-query the list of buckets every time.
+     */
+    private getCheckpointUpdateDynamic;
+}
+export interface CheckpointLine {
+    checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
+    bucketsToFetch: BucketDescription[];
+}
+export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums' | 'getParameterSets'>;

package/dist/sync/BucketChecksumState.js

@@ -0,0 +1,321 @@
+import * as util from '../util/util-index.js';
+import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
+/**
+ * Represents the state of the checksums and data for a specific connection.
+ *
+ * Handles incrementally re-computing checkpoints.
+ */
+export class BucketChecksumState {
+    context;
+    bucketStorage;
+    /**
+     * Bucket state of bucket id -> op_id.
+     * This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
+     */
+    bucketDataPositions = new Map();
+    /**
+     * Last checksums sent to the client. We keep this to calculate checkpoint diffs.
+     */
+    lastChecksums = null;
+    lastWriteCheckpoint = null;
+    parameterState;
+    /**
+     * Keep track of buckets that need to be downloaded. This is specifically relevant when
+     * partial checkpoints are sent.
+     */
+    pendingBucketDownloads = new Set();
+    constructor(options) {
+        this.context = options.syncContext;
+        this.bucketStorage = options.bucketStorage;
+        this.parameterState = new BucketParameterState(options.syncContext, options.bucketStorage, options.syncRules, options.syncParams);
+        this.bucketDataPositions = new Map();
+        for (let { name, after: start } of options.initialBucketPositions ?? []) {
+            this.bucketDataPositions.set(name, { start_op_id: start });
+        }
+    }
+    async buildNextCheckpointLine(next) {
+        const { writeCheckpoint, base } = next;
+        const user_id = this.parameterState.syncParams.user_id;
+        const storage = this.bucketStorage;
+        const update = await this.parameterState.getCheckpointUpdate(next);
+        if (update == null) {
+            return null;
+        }
+        const { buckets: allBuckets, updatedBuckets } = update;
+        let dataBucketsNew = new Map();
+        for (let bucket of allBuckets) {
+            dataBucketsNew.set(bucket.bucket, {
+                description: bucket,
+                start_op_id: this.bucketDataPositions.get(bucket.bucket)?.start_op_id ?? '0'
+            });
+        }
+        this.bucketDataPositions = dataBucketsNew;
+        if (dataBucketsNew.size > this.context.maxBuckets) {
+            throw new ServiceError(ErrorCode.PSYNC_S2305, `Too many buckets: ${dataBucketsNew.size} (limit of ${this.context.maxBuckets})`);
+        }
+        let checksumMap;
+        if (updatedBuckets != null) {
+            if (this.lastChecksums == null) {
+                throw new ServiceAssertionError(`Bucket diff received without existing checksums`);
+            }
+            // Re-check updated buckets only
+            let checksumLookups = [];
+            let newChecksums = new Map();
+            for (let bucket of dataBucketsNew.keys()) {
+                if (!updatedBuckets.has(bucket)) {
+                    const existing = this.lastChecksums.get(bucket);
+                    if (existing == null) {
+                        // If this happens, it means updatedBuckets did not correctly include all new buckets
+                        throw new ServiceAssertionError(`Existing checksum not found for bucket ${bucket}`);
+                    }
+                    // Bucket is not specifically updated, and we have a previous checksum
+                    newChecksums.set(bucket, existing);
+                }
+                else {
+                    checksumLookups.push(bucket);
+                }
+            }
+            let updatedChecksums = await storage.getChecksums(base.checkpoint, checksumLookups);
+            for (let [bucket, value] of updatedChecksums.entries()) {
+                newChecksums.set(bucket, value);
+            }
+            checksumMap = newChecksums;
+        }
+        else {
+            // Re-check all buckets
+            const bucketList = [...dataBucketsNew.keys()];
+            checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
+        }
+        // Subset of buckets for which there may be new data in this batch.
+        let bucketsToFetch;
+        let checkpointLine;
+        if (this.lastChecksums) {
+            // TODO: If updatedBuckets is present, we can use that to more efficiently calculate a diff,
+            // and avoid any unnecessary loops through the entire list of buckets.
+            const diff = util.checksumsDiff(this.lastChecksums, checksumMap);
+            if (this.lastWriteCheckpoint == writeCheckpoint &&
+                diff.removedBuckets.length == 0 &&
+                diff.updatedBuckets.length == 0) {
+                // No changes - don't send anything to the client
+                return null;
+            }
+            let generateBucketsToFetch = new Set();
+            for (let bucket of diff.updatedBuckets) {
+                generateBucketsToFetch.add(bucket.bucket);
+            }
+            for (let bucket of this.pendingBucketDownloads) {
+                // Bucket from a previous checkpoint that hasn't been downloaded yet.
+                // If we still have this bucket, include it in the list of buckets to fetch.
+                if (checksumMap.has(bucket)) {
+                    generateBucketsToFetch.add(bucket);
+                }
+            }
+            const updatedBucketDescriptions = diff.updatedBuckets.map((e) => ({
+                ...e,
+                priority: this.bucketDataPositions.get(e.bucket).description.priority
+            }));
+            bucketsToFetch = [...generateBucketsToFetch].map((b) => {
+                return {
+                    bucket: b,
+                    priority: this.bucketDataPositions.get(b).description.priority
+                };
+            });
+            let message = `Updated checkpoint: ${base.checkpoint} | `;
+            message += `write: ${writeCheckpoint} | `;
+            message += `buckets: ${allBuckets.length} | `;
+            message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
+            message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
+            logger.info(message, {
+                checkpoint: base.checkpoint,
+                user_id: user_id,
+                buckets: allBuckets.length,
+                updated: diff.updatedBuckets.length,
+                removed: diff.removedBuckets.length
+            });
+            checkpointLine = {
+                checkpoint_diff: {
+                    last_op_id: base.checkpoint,
+                    write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
+                    removed_buckets: diff.removedBuckets,
+                    updated_buckets: updatedBucketDescriptions
+                }
+            };
+        }
+        else {
+            let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
+            message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
+            logger.info(message, { checkpoint: base.checkpoint, user_id: user_id, buckets: allBuckets.length });
+            bucketsToFetch = allBuckets;
+            checkpointLine = {
+                checkpoint: {
+                    last_op_id: base.checkpoint,
+                    write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
+                    buckets: [...checksumMap.values()].map((e) => ({
+                        ...e,
+                        priority: this.bucketDataPositions.get(e.bucket).description.priority
+                    }))
+                }
+            };
+        }
+        this.lastChecksums = checksumMap;
+        this.lastWriteCheckpoint = writeCheckpoint;
+        this.pendingBucketDownloads = new Set(bucketsToFetch.map((b) => b.bucket));
+        return {
+            checkpointLine,
+            bucketsToFetch
+        };
+    }
+    /**
+     * Get bucket positions to sync, given the list of buckets.
+     *
+     * @param bucketsToFetch List of buckets to fetch, typically from buildNextCheckpointLine, or a subset of that
+     * @returns
+     */
+    getFilteredBucketPositions(bucketsToFetch) {
+        const filtered = new Map();
+        for (let bucket of bucketsToFetch) {
+            const state = this.bucketDataPositions.get(bucket.bucket);
+            if (state) {
+                filtered.set(bucket.bucket, state.start_op_id);
+            }
+        }
+        return filtered;
+    }
+    /**
+     * Update the position of bucket data the client has.
+     *
+     * @param bucket the bucket name
+     * @param nextAfter sync operations >= this value in the next batch
+     */
+    updateBucketPosition(options) {
+        const state = this.bucketDataPositions.get(options.bucket);
+        if (state) {
+            state.start_op_id = options.nextAfter;
+        }
+        if (!options.hasMore) {
+            this.pendingBucketDownloads.delete(options.bucket);
+        }
+    }
+}
+export class BucketParameterState {
+    context;
+    bucketStorage;
+    syncRules;
+    syncParams;
+    querier;
+    staticBuckets;
+    constructor(context, bucketStorage, syncRules, syncParams) {
+        this.context = context;
+        this.bucketStorage = bucketStorage;
+        this.syncRules = syncRules;
+        this.syncParams = syncParams;
+        this.querier = syncRules.getBucketParameterQuerier(this.syncParams);
+        this.staticBuckets = new Map(this.querier.staticBuckets.map((b) => [b.bucket, b]));
+    }
+    async getCheckpointUpdate(checkpoint) {
+        const querier = this.querier;
+        let update;
+        if (querier.hasDynamicBuckets) {
+            update = await this.getCheckpointUpdateDynamic(checkpoint);
+        }
+        else {
+            update = await this.getCheckpointUpdateStatic(checkpoint);
+        }
+        if (update == null) {
+            return null;
+        }
+        if (update.buckets.length > this.context.maxParameterQueryResults) {
+            // TODO: Limit number of results even before we get to this point
+            // This limit applies _before_ we get the unique set
+            const error = new ServiceError(ErrorCode.PSYNC_S2305, `Too many parameter query results: ${update.buckets.length} (limit of ${this.context.maxParameterQueryResults})`);
+            logger.error(error.message, {
+                checkpoint: checkpoint,
+                user_id: this.syncParams.user_id,
+                buckets: update.buckets.length
+            });
+            throw error;
+        }
+        return update;
+    }
+    /**
+     * For static buckets, we can keep track of which buckets have been updated.
+     */
+    async getCheckpointUpdateStatic(checkpoint) {
+        const querier = this.querier;
+        const update = checkpoint.update;
+        if (update.invalidateDataBuckets) {
+            return {
+                buckets: querier.staticBuckets,
+                updatedBuckets: null
+            };
+        }
+        let updatedBuckets = new Set();
+        for (let bucket of update.updatedDataBuckets ?? []) {
+            if (this.staticBuckets.has(bucket)) {
+                updatedBuckets.add(bucket);
+            }
+        }
+        if (updatedBuckets.size == 0) {
+            // No change - skip this checkpoint
+            return null;
+        }
+        return {
+            buckets: querier.staticBuckets,
+            updatedBuckets
+        };
+    }
+    /**
+     * For dynamic buckets, we need to re-query the list of buckets every time.
+     */
+    async getCheckpointUpdateDynamic(checkpoint) {
+        const querier = this.querier;
+        const storage = this.bucketStorage;
+        const staticBuckets = querier.staticBuckets;
+        const update = checkpoint.update;
+        let hasChange = false;
+        if (update.invalidateDataBuckets || update.updatedDataBuckets?.length > 0) {
+            hasChange = true;
+        }
+        else if (update.invalidateParameterBuckets) {
+            hasChange = true;
+        }
+        else {
+            for (let bucket of update.updatedParameterBucketDefinitions ?? []) {
+                if (querier.dynamicBucketDefinitions.has(bucket)) {
+                    hasChange = true;
+                    break;
+                }
+            }
+        }
+        if (!hasChange) {
+            return null;
+        }
+        const dynamicBuckets = await querier.queryDynamicBucketDescriptions({
+            getParameterSets(lookups) {
+                return storage.getParameterSets(checkpoint.base.checkpoint, lookups);
+            }
+        });
+        const allBuckets = [...staticBuckets, ...dynamicBuckets];
+        return {
+            buckets: allBuckets,
+            // We cannot track individual bucket updates for dynamic lookups yet
+            updatedBuckets: null
+        };
+    }
+}
+function limitedBuckets(buckets, limit) {
+    buckets = buckets.map((b) => {
+        if (typeof b != 'string') {
+            return b.bucket;
+        }
+        else {
+            return b;
+        }
+    });
+    if (buckets.length <= limit) {
+        return JSON.stringify(buckets);
+    }
+    const limited = buckets.slice(0, limit);
+    return `${JSON.stringify(limited)}...`;
+}
+//# sourceMappingURL=BucketChecksumState.js.map

package/dist/sync/BucketChecksumState.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BucketChecksumState.js","sourceRoot":"","sources":["../../src/sync/BucketChecksumState.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAa3G;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAc;IACrB,aAAa,CAA6B;IAE3D;;;OAGG;IACI,mBAAmB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAEhE;;OAEG;IACK,aAAa,GAA4B,IAAI,CAAC;IAC9C,mBAAmB,GAAkB,IAAI,CAAC;IAEjC,cAAc,CAAuB;IAEtD;;;OAGG;IACK,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnD,YAAY,OAAmC;QAC7C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAC5C,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,aAAa,EACrB,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,UAAU,CACnB,CAAC;QACF,IAAI,CAAC,mBAAmB,GAAG,IAAI,GAAG,EAAE,CAAC;QAErC,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,OAAO,CAAC,sBAAsB,IAAI,EAAE,EAAE,CAAC;YACxE,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,IAAqC;QACjE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC;QAEvD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC;QAEnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QACnE,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;QAEvD,IAAI,cAAc,GAAG,IAAI,GAAG,EAA2B,CAAC;QACxD,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YAC9B,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;gBAChC,WAAW,EAAE,MAAM;gBACnB,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,GAAG;aAC7E,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC;QAC1C,IAAI,cAAc,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YAClD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,qBAAqB,cAAc,CAAC,IAAI,cAAc,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,CACjF,CAAC;QACJ,CAAC;QAED,IAAI,WAA6B,CAAC;QAClC,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;gBAC/B,MAAM,IAAI,qBAAqB,CAAC,iDAAiD,CAAC,CAAC;YACrF,CAAC;YAED,gCAAgC;YAChC,IAAI,eAAe,GAAa,EAAE,CAAC;YAEnC,IAAI,YAAY,GAAG,IAAI,GAAG,EAA+B,CAAC;YAC1D,KAAK,IAAI,MAAM,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAChD,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;wBACrB,qFAAqF;wBACrF,MAAM,IAAI,qBAAqB,CAAC,0CAA0C,MAAM,EAAE,CAAC,CAAC;oBACtF,CAAC;oBACD,sEAAsE;oBACtE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACrC,CAAC;qBAAM,CAAC;oBACN,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,IAAI,gBAAgB,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;YACpF,KAAK,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,gBAAgB,CAAC,OAAO,EAAE,EAAE,CAAC;gBACvD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,WAAW,GAAG,YAAY,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,uBAAuB;YACvB,MAAM,UAAU,GAAG,CAAC,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9C,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACxE,CAAC;QACD,mEAAmE;QACnE,IAAI,cAAmC,CAAC;QAExC,IAAI,cAA+E,CAAC;QAEpF,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,4FAA4F;YAC5F,sEAAsE;YACtE,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAEjE,IACE,IAAI,CAAC,mBAAmB,IAAI,eAAe;gBAC3C,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B,CAAC;gBACD,iDAAiD;gBACjD,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/C,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACvC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,CAAC;YACD,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC/C,qEAAqE;gBACrE,4EAA4E;gBAC5E,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC5B,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;YAED,MAAM,yBAAyB,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAChE,GAAG,CAAC;gBACJ,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAE,CAAC,WAAY,CAAC,QAAQ;aACxE,CAAC,CAAC,CAAC;YACJ,cAAc,GAAG,CAAC,GAAG,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACrD,OAAO;oBACL,MAAM,EAAE,CAAC;oBACT,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,WAAY,CAAC,QAAQ;iBACjE,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,IAAI,OAAO,GAAG,uBAAuB,IAAI,CAAC,UAAU,KAAK,CAAC;YAC1D,OAAO,IAAI,UAAU,eAAe,KAAK,CAAC;YAC1C,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,UAAU,CAAC,MAAM;gBAC1B,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;gBACnC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;aACpC,CAAC,CAAC;YAEH,cAAc,GAAG;gBACf,eAAe,EAAE;oBACf,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,yBAAyB;iBAC3C;aACyC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,IAAI,OAAO,GAAG,mBAAmB,IAAI,CAAC,UAAU,aAAa,eAAe,KAAK,CAAC;YAClF,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;YACpG,cAAc,GAAG,UAAU,CAAC;YAC5B,cAAc,GAAG;gBACf,UAAU,EAAE;oBACV,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC7C,GAAG,CAAC;wBACJ,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAE,CAAC,WAAY,CAAC,QAAQ;qBACxE,CAAC,CAAC;iBACJ;aACqC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC;QACjC,IAAI,CAAC,mBAAmB,GAAG,eAAe,CAAC;QAC3C,IAAI,CAAC,sBAAsB,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3E,OAAO;YACL,cAAc;YACd,cAAc;SACf,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,0BAA0B,CAAC,cAAmC;QAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,KAAK,IAAI,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;;OAKG;IACH,oBAAoB,CAAC,OAAgE;QACnF,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3D,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC;QACxC,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;CACF;AAgBD,MAAM,OAAO,oBAAoB;IACd,OAAO,CAAc;IACtB,aAAa,CAA6B;IAC1C,SAAS,CAAe;IACxB,UAAU,CAAoB;IAC7B,OAAO,CAAyB;IAChC,aAAa,CAAiC;IAE/D,YACE,OAAoB,EACpB,aAAyC,EACzC,SAAuB,EACvB,UAA6B;QAE7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAE7B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,yBAAyB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpE,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,CAA4B,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAChH,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,UAA2C;QACnE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,MAA+B,CAAC;QACpC,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAC9B,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;QAC5D,CAAC;QAED,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC;YAClE,iEAAiE;YACjE,oDAAoD;YACpD,MAAM,KAAK,GAAG,IAAI,YAAY,CAC5B,SAAS,CAAC,WAAW,EACrB,qCAAqC,MAAM,CAAC,OAAO,CAAC,MAAM,cAAc,IAAI,CAAC,OAAO,CAAC,wBAAwB,GAAG,CACjH,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE;gBAC1B,UAAU,EAAE,UAAU;gBACtB,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO;gBAChC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;aAC/B,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,yBAAyB,CACrC,UAA2C;QAE3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;QAEjC,IAAI,MAAM,CAAC,qBAAqB,EAAE,CAAC;YACjC,OAAO;gBACL,OAAO,EAAE,OAAO,CAAC,aAAa;gBAC9B,cAAc,EAAE,IAAI;aACrB,CAAC;QACJ,CAAC;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,KAAK,IAAI,MAAM,IAAI,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;YACnD,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC7B,mCAAmC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,cAAc;SACf,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,0BAA0B,CACtC,UAA2C;QAE3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC;QACnC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;QAEjC,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,MAAM,CAAC,qBAAqB,IAAI,MAAM,CAAC,kBAAkB,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1E,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;aAAM,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;YAC7C,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,MAAM,IAAI,MAAM,CAAC,iCAAiC,IAAI,EAAE,EAAE,CAAC;gBAClE,IAAI,OAAO,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjD,SAAS,GAAG,IAAI,CAAC;oBACjB,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,8BAA8B,CAAC;YAClE,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvE,CAAC;SACF,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,CAAC,GAAG,aAAa,EAAE,GAAG,cAAc,CAAC,CAAC;QAEzD,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,oEAAoE;YACpE,cAAc,EAAE,IAAI;SACrB,CAAC;IACJ,CAAC;CACF;AAUD,SAAS,cAAc,CAAC,OAAwC,EAAE,KAAa;IAC7E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC,MAAM,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}

package/dist/sync/SyncContext.d.ts

@@ -0,0 +1,17 @@
+import { SemaphoreInterface } from 'async-mutex';
+export interface SyncContextOptions {
+    maxBuckets: number;
+    maxParameterQueryResults: number;
+    maxDataFetchConcurrency: number;
+}
+/**
+ * Represents the context in which sync happens.
+ *
+ * This is global to all sync requests, not per request.
+ */
+export declare class SyncContext {
+    readonly maxBuckets: number;
+    readonly maxParameterQueryResults: number;
+    readonly syncSemaphore: SemaphoreInterface;
+    constructor(options: SyncContextOptions);
+}

package/dist/sync/SyncContext.js

@@ -0,0 +1,23 @@
+import { Semaphore, withTimeout } from 'async-mutex';
+/**
+ * Maximum duration to wait for the mutex to become available.
+ *
+ * This gives an explicit error if there are mutex issues, rather than just hanging.
+ */
+const MUTEX_ACQUIRE_TIMEOUT = 30_000;
+/**
+ * Represents the context in which sync happens.
+ *
+ * This is global to all sync requests, not per request.
+ */
+export class SyncContext {
+    maxBuckets;
+    maxParameterQueryResults;
+    syncSemaphore;
+    constructor(options) {
+        this.maxBuckets = options.maxBuckets;
+        this.maxParameterQueryResults = options.maxParameterQueryResults;
+        this.syncSemaphore = withTimeout(new Semaphore(options.maxDataFetchConcurrency), MUTEX_ACQUIRE_TIMEOUT, new Error(`Timeout while waiting for data`));
+    }
+}
+//# sourceMappingURL=SyncContext.js.map

package/dist/sync/SyncContext.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SyncContext.js","sourceRoot":"","sources":["../../src/sync/SyncContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAsB,WAAW,EAAE,MAAM,aAAa,CAAC;AAQzE;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC;;;;GAIG;AACH,MAAM,OAAO,WAAW;IACb,UAAU,CAAS;IACnB,wBAAwB,CAAS;IAEjC,aAAa,CAAqB;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,wBAAwB,CAAC;QACjE,IAAI,CAAC,aAAa,GAAG,WAAW,CAC9B,IAAI,SAAS,CAAC,OAAO,CAAC,uBAAuB,CAAC,EAC9C,qBAAqB,EACrB,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAC5C,CAAC;IACJ,CAAC;CACF"}

package/dist/sync/sync-index.d.ts

@@ -5,3 +5,5 @@ export * from './RequestTracker.js';
 export * from './safeRace.js';
 export * from './sync.js';
 export * from './util.js';
+export * from './BucketChecksumState.js';
+export * from './SyncContext.js';

package/dist/sync/sync-index.js

@@ -5,4 +5,6 @@ export * from './RequestTracker.js';
 export * from './safeRace.js';
 export * from './sync.js';
 export * from './util.js';
+export * from './BucketChecksumState.js';
+export * from './SyncContext.js';
 //# sourceMappingURL=sync-index.js.map

package/dist/sync/sync-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"sync-index.js","sourceRoot":"","sources":["../../src/sync/sync-index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,WAAW,CAAC;AAC1B,cAAc,WAAW,CAAC"}
+{"version":3,"file":"sync-index.js","sourceRoot":"","sources":["../../src/sync/sync-index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,WAAW,CAAC;AAC1B,cAAc,WAAW,CAAC;AAC1B,cAAc,0BAA0B,CAAC;AACzC,cAAc,kBAAkB,CAAC"}

package/dist/sync/sync.d.ts

@@ -1,15 +1,17 @@
-import { RequestParameters } from '@powersync/service-sync-rules';
+import { BucketDescription, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as auth from '../auth/auth-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
-import { RequestTracker } from './RequestTracker.js';
 import { TokenStreamOptions } from './util.js';
+import { SyncContext } from './SyncContext.js';
+import { RequestTracker } from './RequestTracker.js';
 export interface SyncStreamParameters {
-    storage: storage.BucketStorageFactory;
+    syncContext: SyncContext;
+    bucketStorage: storage.SyncRulesBucketStorage;
+    syncRules: SqlSyncRules;
     params: util.StreamingSyncRequest;
     syncParams: RequestParameters;
     token: auth.JwtPayload;
-    parseOptions: storage.ParseSyncRulesOptions;
     /**
      * If this signal is aborted, the stream response ends as soon as possible, without error.
      */
@@ -18,3 +20,7 @@ export interface SyncStreamParameters {
     tracker: RequestTracker;
 }
 export declare function streamResponse(options: SyncStreamParameters): AsyncIterable<util.StreamingSyncLine | string | null>;
+export type BucketSyncState = {
+    description?: BucketDescription;
+    start_op_id: string;
+};