@powersync/service-core 0.2.0 → 0.2.1

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @powersync/service-core
 
+## 0.2.1
+
+### Patch Changes
+
+- 4a57787:
+  - Use a LRU cache for checksum computations, improving performance and reducing MongoDB database load.
+  - Return zero checksums to the client instead of omitting, to help with debugging sync issues.
+
 ## 0.2.0
 
 ### Minor Changes

package/dist/storage/BucketStorage.d.ts

@@ -176,7 +176,12 @@ export interface SyncRulesBucketStorage {
      * @param options batch size options
      */
     getBucketDataBatch(checkpoint: util.OpId, dataBuckets: Map<string, string>, options?: BucketDataBatchOptions): AsyncIterable<util.SyncBucketData>;
-    getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.BucketChecksum[]>;
+    /**
+     * Compute checksums for a given list of buckets.
+     *
+     * Returns zero checksums for any buckets not found.
+     */
+    getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
     /**
      * Terminate the sync rules.
      *

package/dist/storage/BucketStorage.js.map

@@ -1 +1 @@
-{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AA+JA,MAAM,OAAO,yBAAyB;IAGpC,YAA4B,EAAU,EAAkB,UAAwB,EAAE,cAA6B;QAAnF,OAAE,GAAF,EAAE,CAAQ;QAAkB,eAAU,GAAV,UAAU,CAAc;QAC9E,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;IAED,IAAI,SAAS;QACX,OAAO,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC;IAChC,CAAC;CACF;AAYD,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAsMlE,MAAM,UAAU,UAAU,CAAC,MAA0B,EAAE,SAA6B;IAClF,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACtC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
+{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AA+JA,MAAM,OAAO,yBAAyB;IAGpC,YAA4B,EAAU,EAAkB,UAAwB,EAAE,cAA6B;QAAnF,OAAE,GAAF,EAAE,CAAQ;QAAkB,eAAU,GAAV,UAAU,CAAc;QAC9E,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;IAED,IAAI,SAAS;QACX,OAAO,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC;IAChC,CAAC;CACF;AAYD,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AA2MlE,MAAM,UAAU,UAAU,CAAC,MAA0B,EAAE,SAA6B;IAClF,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACtC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/storage/ChecksumCache.d.ts

@@ -0,0 +1,50 @@
+import { BucketChecksum, OpId } from '../util/protocol-types.js';
+import { ChecksumMap } from '../util/utils.js';
+export interface FetchPartialBucketChecksum {
+    bucket: string;
+    start?: OpId;
+    end: OpId;
+}
+export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<ChecksumMap>;
+export interface ChecksumCacheOptions {
+    /**
+     * Upstream checksum implementation.
+     *
+     * This fetches a batch of either entire bucket checksums, or a partial range.
+     */
+    fetchChecksums: FetchChecksums;
+    /**
+     * Maximum number of cached checksums.
+     */
+    maxSize?: number;
+}
+/**
+ * Implement a LRU cache for checksum requests. Each (bucket, checkpoint) request is cached separately,
+ * while the lookups occur in batches.
+ *
+ * For each bucket, we keep a separate OrderedSet of cached checkpoints.
+ * This allows us to do incrementally update checksums by using the last cached checksum for the same bucket.
+ *
+ * We use the LRUCache fetchMethod to deduplicate in-progress requests.
+ */
+export declare class ChecksumCache {
+    /**
+     * The primary checksum cache, with key of `${checkpoint}/${bucket}`.
+     */
+    private cache;
+    /**
+     * For each bucket, an ordered set of cached checkpoints.
+     */
+    private bucketCheckpoints;
+    private fetchChecksums;
+    constructor(options: ChecksumCacheOptions);
+    getChecksums(checkpoint: OpId, buckets: string[]): Promise<BucketChecksum[]>;
+    /**
+     * Get bucket checksums for a checkpoint.
+     *
+     * Any checksums not found upstream are returned as zero checksums.
+     *
+     * @returns a Map with exactly one entry for each bucket requested
+     */
+    getChecksumMap(checkpoint: OpId, buckets: string[]): Promise<ChecksumMap>;
+}

package/dist/storage/ChecksumCache.js

@@ -0,0 +1,234 @@
+import { addBucketChecksums } from '../util/utils.js';
+import { LRUCache } from 'lru-cache/min';
+import { OrderedSet } from '@js-sdsl/ordered-set';
+// Approximately 5MB of memory, if we assume 50 bytes per entry
+const DEFAULT_MAX_SIZE = 100000;
+/**
+ * Implement a LRU cache for checksum requests. Each (bucket, checkpoint) request is cached separately,
+ * while the lookups occur in batches.
+ *
+ * For each bucket, we keep a separate OrderedSet of cached checkpoints.
+ * This allows us to do incrementally update checksums by using the last cached checksum for the same bucket.
+ *
+ * We use the LRUCache fetchMethod to deduplicate in-progress requests.
+ */
+export class ChecksumCache {
+    constructor(options) {
+        /**
+         * For each bucket, an ordered set of cached checkpoints.
+         */
+        this.bucketCheckpoints = new Map();
+        this.fetchChecksums = options.fetchChecksums;
+        this.cache = new LRUCache({
+            max: options.maxSize ?? DEFAULT_MAX_SIZE,
+            fetchMethod: async (cacheKey, _staleValue, options) => {
+                // Called when this checksum hasn't been cached yet.
+                // Pass the call back to the request, which implements batch fetching.
+                const { bucket } = parseCacheKey(cacheKey);
+                const result = await options.context.fetch(bucket);
+                // Add to the set of cached checkpoints for the bucket.
+                let checkpointSet = this.bucketCheckpoints.get(bucket);
+                if (checkpointSet == null) {
+                    checkpointSet = new OrderedSet();
+                    this.bucketCheckpoints.set(bucket, checkpointSet);
+                }
+                checkpointSet.insert(options.context.checkpoint);
+                return result;
+            },
+            dispose: (value, key) => {
+                // Remove from the set of cached checkpoints for the bucket
+                const { checkpointString } = parseCacheKey(key);
+                const checkpoint = BigInt(checkpointString);
+                const checkpointSet = this.bucketCheckpoints.get(value.bucket);
+                if (checkpointSet == null) {
+                    return;
+                }
+                checkpointSet.eraseElementByKey(checkpoint);
+                if (checkpointSet.length == 0) {
+                    this.bucketCheckpoints.delete(value.bucket);
+                }
+            },
+            noDisposeOnSet: true,
+            // When we have more fetches than the cache size, complete the fetches instead
+            // of failing with Error('evicted').
+            ignoreFetchAbort: true
+        });
+    }
+    async getChecksums(checkpoint, buckets) {
+        const checksums = await this.getChecksumMap(checkpoint, buckets);
+        // Return results in the same order as the request
+        return buckets.map((bucket) => checksums.get(bucket));
+    }
+    /**
+     * Get bucket checksums for a checkpoint.
+     *
+     * Any checksums not found upstream are returned as zero checksums.
+     *
+     * @returns a Map with exactly one entry for each bucket requested
+     */
+    async getChecksumMap(checkpoint, buckets) {
+        // Buckets that don't have a cached checksum for this checkpoint yet
+        let toFetch = new Set();
+        // Newly fetched results
+        let fetchResults = new Map();
+        // Promise for the bactch new fetch requests
+        let resolveFetch;
+        let rejectFetch;
+        let fetchPromise = new Promise((resolve, reject) => {
+            resolveFetch = resolve;
+            rejectFetch = reject;
+        });
+        // Accumulated results - both from cached checksums, and fetched checksums
+        let finalResults = new Map();
+        const context = {
+            async fetch(bucket) {
+                await fetchPromise;
+                if (!toFetch.has(bucket)) {
+                    // Should never happen
+                    throw new Error(`Expected to fetch ${bucket}`);
+                }
+                const checksum = fetchResults.get(bucket);
+                if (checksum == null) {
+                    // Should never happen
+                    throw new Error(`Failed to fetch checksum for bucket ${bucket}`);
+                }
+                return checksum;
+            },
+            checkpoint: BigInt(checkpoint)
+        };
+        // One promise to await to ensure all fetch requests completed.
+        let settledPromise = null;
+        try {
+            // Individual cache fetch promises
+            let cacheFetchPromises = [];
+            for (let bucket of buckets) {
+                const cacheKey = makeCacheKey(checkpoint, bucket);
+                let status = {};
+                const p = this.cache.fetch(cacheKey, { context: context, status: status }).then((checksums) => {
+                    if (checksums == null) {
+                        // Should never happen
+                        throw new Error(`Failed to get checksums for ${cacheKey}`);
+                    }
+                    finalResults.set(bucket, checksums);
+                });
+                cacheFetchPromises.push(p);
+                if (status.fetch == 'hit' || status.fetch == 'inflight') {
+                    // The checksums is either cached already (hit), or another request is busy
+                    // fetching (inflight).
+                    // In either case, we don't need to fetch a new checksum.
+                }
+                else {
+                    // We need a new request for this checksum.
+                    toFetch.add(bucket);
+                }
+            }
+            // We do this directly after creating the promises, otherwise
+            // we could end up with weird uncaught rejection errors.
+            settledPromise = Promise.allSettled(cacheFetchPromises);
+            if (toFetch.size == 0) {
+                // Nothing to fetch, but resolve in case
+                resolveFetch();
+            }
+            else {
+                let bucketRequests = [];
+                // Partial checksum (previously cached) to add to the partial fetch
+                let add = new Map();
+                for (let bucket of toFetch) {
+                    let bucketRequest = null;
+                    const checkpointSet = this.bucketCheckpoints.get(bucket);
+                    if (checkpointSet != null) {
+                        // Find smaller checkpoints, sorted in descending order
+                        let iter = checkpointSet.reverseUpperBound(context.checkpoint);
+                        const begin = checkpointSet.begin();
+                        while (iter.isAccessible()) {
+                            const cp = iter.pointer;
+                            const cacheKey = makeCacheKey(cp, bucket);
+                            // peek to avoid refreshing the key
+                            const cached = this.cache.peek(cacheKey);
+                            // As long as dispose() works correctly, the checkpointset should
+                            // match up with the cache, and `cached` should also have a value here.
+                            // However, we handle caces where it's not present either way.
+                            // Test by disabling the `dispose()` callback.
+                            if (cached != null) {
+                                // Partial checksum found - make a partial checksum request
+                                bucketRequest = {
+                                    bucket,
+                                    start: cp.toString(),
+                                    end: checkpoint
+                                };
+                                add.set(bucket, cached);
+                                break;
+                            }
+                            if (iter.equals(begin)) {
+                                // Cannot iterate further
+                                break;
+                            }
+                            // Iterate backwards
+                            iter = iter.pre();
+                        }
+                    }
+                    if (bucketRequest == null) {
+                        // No partial checksum found - make a new full checksum request
+                        bucketRequest = {
+                            bucket,
+                            end: checkpoint
+                        };
+                        add.set(bucket, {
+                            bucket,
+                            checksum: 0,
+                            count: 0
+                        });
+                    }
+                    bucketRequests.push(bucketRequest);
+                }
+                // Fetch partial checksums from upstream
+                const results = await this.fetchChecksums(bucketRequests);
+                for (let bucket of toFetch) {
+                    const result = results.get(bucket);
+                    const toAdd = add.get(bucket);
+                    if (toAdd == null) {
+                        // Should never happen
+                        throw new Error(`toAdd null for ${bucket}`);
+                    }
+                    // Compute the full checksum from the two partials.
+                    // No results returned are treated the same as a zero result.
+                    const added = addBucketChecksums(toAdd, result ?? null);
+                    fetchResults.set(bucket, added);
+                }
+                // fetchResults is fully populated, so we resolve the Promise
+                resolveFetch();
+            }
+        }
+        catch (e) {
+            // Failure when fetching checksums - reject the Promise.
+            // This will reject all individual cache fetch requests, and each will be retried
+            // on the next request.
+            rejectFetch(e);
+            // Wait for the above rejection to propagate, otherwise we end up with "uncaught" errors.
+            // This promise never throws.
+            await settledPromise;
+            throw e;
+        }
+        // Wait for all cache fetch reqeusts to complete
+        const settledResults = (await settledPromise) ?? [];
+        // Check if any of them failed
+        for (let result of settledResults) {
+            if (result.status == 'rejected') {
+                throw result.reason;
+            }
+        }
+        if (finalResults.size != buckets.length) {
+            // Should not happen
+            throw new Error(`Bucket results mismatch: ${finalResults.size} != ${buckets.length}`);
+        }
+        return finalResults;
+    }
+}
+function makeCacheKey(checkpoint, bucket) {
+    return `${checkpoint}/${bucket}`;
+}
+function parseCacheKey(key) {
+    const index = key.indexOf('/');
+    return { checkpointString: key.substring(0, index), bucket: key.substring(index + 1) };
+}
+//# sourceMappingURL=ChecksumCache.js.map

package/dist/storage/ChecksumCache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChecksumCache.js","sourceRoot":"","sources":["../../src/storage/ChecksumCache.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AA6BlD,+DAA+D;AAC/D,MAAM,gBAAgB,GAAG,MAAO,CAAC;AAEjC;;;;;;;;GAQG;AACH,MAAM,OAAO,aAAa;IAYxB,YAAY,OAA6B;QAPzC;;WAEG;QACK,sBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAC;QAKhE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAA+C;YACtE,GAAG,EAAE,OAAO,CAAC,OAAO,IAAI,gBAAgB;YACxC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE;gBACpD,oDAAoD;gBACpD,sEAAsE;gBACtE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAEnD,uDAAuD;gBACvD,IAAI,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACvD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,aAAa,GAAG,IAAI,UAAU,EAAE,CAAC;oBACjC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;gBACpD,CAAC;gBACD,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACjD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACtB,2DAA2D;gBAC3D,MAAM,EAAE,gBAAgB,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;gBAChD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;gBAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBACD,aAAa,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;gBAC5C,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,cAAc,EAAE,IAAI;YAEpB,8EAA8E;YAC9E,oCAAoC;YACpC,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAgB,EAAE,OAAiB;QACpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACjE,kDAAkD;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,UAAgB,EAAE,OAAiB;QACtD,oEAAoE;QACpE,IAAI,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAEhC,wBAAwB;QACxB,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,4CAA4C;QAC5C,IAAI,YAAyB,CAAC;QAC9B,IAAI,WAAgC,CAAC;QACrC,IAAI,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,YAAY,GAAG,OAAO,CAAC;YACvB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,MAAM,OAAO,GAAyB;YACpC,KAAK,CAAC,KAAK,CAAC,MAAM;gBAChB,MAAM,YAAY,CAAC;gBACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;gBACD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAC;gBACnE,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;SAC/B,CAAC;QAEF,+DAA+D;QAC/D,IAAI,cAAc,GAAiD,IAAI,CAAC;QAExE,IAAI,CAAC;YACH,kCAAkC;YAClC,IAAI,kBAAkB,GAAoB,EAAE,CAAC;YAE7C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClD,IAAI,MAAM,GAAoC,EAAE,CAAC;gBACjD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;oBAC5F,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;wBACtB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;oBAC7D,CAAC;oBACD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBACH,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,IAAI,MAAM,CAAC,KAAK,IAAI,KAAK,IAAI,MAAM,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxD,2EAA2E;oBAC3E,uBAAuB;oBACvB,yDAAyD;gBAC3D,CAAC;qBAAM,CAAC;oBACN,2CAA2C;oBAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,6DAA6D;YAC7D,wDAAwD;YACxD,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAExD,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;gBACtB,wCAAwC;gBACxC,YAAY,EAAE,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,IAAI,cAAc,GAAiC,EAAE,CAAC;gBACtD,mEAAmE;gBACnE,IAAI,GAAG,GAAG,IAAI,GAAG,EAA0B,CAAC;gBAE5C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,aAAa,GAAsC,IAAI,CAAC;oBAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACzD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,uDAAuD;wBACvD,IAAI,IAAI,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;wBAC/D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC;wBACpC,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;4BAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;4BACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;4BAC1C,mCAAmC;4BACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;4BACzC,iEAAiE;4BACjE,uEAAuE;4BACvE,8DAA8D;4BAC9D,8CAA8C;4BAC9C,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gCACnB,2DAA2D;gCAC3D,aAAa,GAAG;oCACd,MAAM;oCACN,KAAK,EAAE,EAAE,CAAC,QAAQ,EAAE;oCACpB,GAAG,EAAE,UAAU;iCAChB,CAAC;gCACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gCACxB,MAAM;4BACR,CAAC;4BAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gCACvB,yBAAyB;gCACzB,MAAM;4BACR,CAAC;4BACD,oBAAoB;4BACpB,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACpB,CAAC;oBACH,CAAC;oBAED,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,+DAA+D;wBAC/D,aAAa,GAAG;4BACd,MAAM;4BACN,GAAG,EAAE,UAAU;yBAChB,CAAC;wBACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;4BACd,MAAM;4BACN,QAAQ,EAAE,CAAC;4BACX,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBACrC,CAAC;gBAED,wCAAwC;gBACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;gBAE1D,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;wBAClB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,EAAE,CAAC,CAAC;oBAC9C,CAAC;oBACD,mDAAmD;oBACnD,6DAA6D;oBAC7D,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;oBACxD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBAClC,CAAC;gBAED,6DAA6D;gBAC7D,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,wDAAwD;YACxD,iFAAiF;YACjF,uBAAuB;YACvB,WAAW,CAAC,CAAC,CAAC,CAAC;YAEf,yFAAyF;YACzF,6BAA6B;YAC7B,MAAM,cAAc,CAAC;YAErB,MAAM,CAAC,CAAC;QACV,CAAC;QAED,gDAAgD;QAChD,MAAM,cAAc,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACpD,8BAA8B;QAC9B,KAAK,IAAI,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACxC,oBAAoB;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,YAAY,CAAC,IAAI,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;CACF;AAED,SAAS,YAAY,CAAC,UAA2B,EAAE,MAAc;IAC/D,OAAO,GAAG,UAAU,IAAI,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,EAAE,gBAAgB,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;AACzF,CAAC"}

package/dist/storage/mongo/MongoSyncBucketStorage.d.ts

@@ -8,6 +8,7 @@ export declare class MongoSyncBucketStorage implements SyncRulesBucketStorage {
     readonly sync_rules: SqlSyncRules;
     readonly slot_name: string;
     private readonly db;
+    private checksumCache;
     constructor(factory: MongoBucketStorage, group_id: number, sync_rules: SqlSyncRules, slot_name: string);
     getCheckpoint(): Promise<{
         checkpoint: string;
@@ -17,7 +18,8 @@ export declare class MongoSyncBucketStorage implements SyncRulesBucketStorage {
     resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
     getParameterSets(checkpoint: util.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]>;
     getBucketDataBatch(checkpoint: util.OpId, dataBuckets: Map<string, string>, options?: BucketDataBatchOptions): AsyncIterable<util.SyncBucketData>;
-    getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.BucketChecksum[]>;
+    getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
+    private getChecksumsInternal;
     terminate(): Promise<void>;
     getStatus(): Promise<SyncRuleStatus>;
     clear(): Promise<void>;

package/dist/storage/mongo/MongoSyncBucketStorage.js

@@ -7,12 +7,18 @@ import { SourceTable } from '../SourceTable.js';
 import { SyncRuleState } from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, readSingleBatch, serializeLookup } from './util.js';
+import { ChecksumCache } from '../ChecksumCache.js';
 export class MongoSyncBucketStorage {
     constructor(factory, group_id, sync_rules, slot_name) {
         this.factory = factory;
         this.group_id = group_id;
         this.sync_rules = sync_rules;
         this.slot_name = slot_name;
+        this.checksumCache = new ChecksumCache({
+            fetchChecksums: (batch) => {
+                return this.getChecksumsInternal(batch);
+            }
+        });
         this.db = factory.db;
     }
     async getCheckpoint() {
@@ -231,22 +237,25 @@ export class MongoSyncBucketStorage {
         }
     }
     async getChecksums(checkpoint, buckets) {
-        if (buckets.length == 0) {
-            return [];
+        return this.checksumCache.getChecksumMap(checkpoint, buckets);
+    }
+    async getChecksumsInternal(batch) {
+        if (batch.length == 0) {
+            return new Map();
         }
         const filters = [];
-        for (let name of buckets) {
+        for (let request of batch) {
             filters.push({
                 _id: {
                     $gt: {
                         g: this.group_id,
-                        b: name,
-                        o: new bson.MinKey()
+                        b: request.bucket,
+                        o: request.start ? BigInt(request.start) : new bson.MinKey()
                     },
                     $lte: {
                         g: this.group_id,
-                        b: name,
-                        o: BigInt(checkpoint)
+                        b: request.bucket,
+                        o: BigInt(request.end)
                     }
                 }
             });
@@ -263,13 +272,16 @@ export class MongoSyncBucketStorage {
             }
         ], { session: undefined })
             .toArray();
-        return aggregate.map((doc) => {
-            return {
-                bucket: doc._id,
-                count: doc.count,
-                checksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 4294967295
-            };
-        });
+        return new Map(aggregate.map((doc) => {
+            return [
+                doc._id,
+                {
+                    bucket: doc._id,
+                    count: doc.count,
+                    checksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff
+                }
+            ];
+        }));
     }
     async terminate() {
         await this.clear();

package/dist/storage/mongo/MongoSyncBucketStorage.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoSyncBucketStorage.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoSyncBucketStorage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,WAAW,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAGL,4BAA4B,EAC5B,kCAAkC,EAMnC,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAgD,aAAa,EAAE,MAAM,aAAa,CAAC;AAC1F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEvG,MAAM,OAAO,sBAAsB;IAGjC,YACkB,OAA2B,EAC3B,QAAgB,EAChB,UAAwB,EACxB,SAAiB;QAHjB,YAAO,GAAP,OAAO,CAAoB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,eAAU,GAAV,UAAU,CAAc;QACxB,cAAS,GAAT,SAAS,CAAQ;QAEjC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EACtB;YACE,UAAU,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE;SAC3D,CACF,CAAC;QACF,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,eAAe,IAAI,EAAE,CAAC;YAC5D,GAAG,EAAE,GAAG,EAAE,mBAAmB,IAAI,WAAW,CAAC,QAAQ;SACtD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAW,EAAE,QAAsD;QAClF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD,EAAE,UAAU,EAAE,EAAE,mBAAmB,EAAE,CAAC,EAAE,oBAAoB,EAAE,CAAC,EAAE,EAAE,CACpE,CAAC;QACF,MAAM,cAAc,GAAG,GAAG,EAAE,mBAAmB,IAAI,IAAI,CAAC;QAExD,MAAM,KAAK,GAAG,IAAI,gBAAgB,CAChC,IAAI,CAAC,EAAE,EACP,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,SAAS,EACd,cAAc,EACd,GAAG,EAAE,oBAAoB,IAAI,IAAI,CAClC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC7C,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAEtE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,GAAG,QAAQ,CAAC;QAEzE,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACtG,IAAI,MAAM,GAA8B,IAAI,CAAC;QAC7C,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC;YAClC,IAAI,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CACzB;gBACE,QAAQ,EAAE,QAAQ;gBAClB,aAAa,EAAE,aAAa;gBAC5B,WAAW,EAAE,UAAU;gBACvB,WAAW,EAAE,MAAM;gBACnB,UAAU,EAAE,KAAK;gBACjB,mBAAmB,EAAE,OAAO;aAC7B,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;YACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,GAAG,GAAG;oBACJ,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE;oBACxB,QAAQ,EAAE,QAAQ;oBAClB,aAAa,EAAE,aAAa;oBAC5B,WAAW,EAAE,UAAU;oBACvB,WAAW,EAAE,MAAM;oBACnB,UAAU,EAAE,KAAK;oBACjB,kBAAkB,EAAE,IAAI;oBACxB,mBAAmB,EAAE,OAAO;oBAC5B,aAAa,EAAE,KAAK;iBACrB,CAAC;gBAEF,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACxC,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,GAAG,CAAC,GAAG,EACP,cAAc,EACd,UAAU,EACV,MAAM,EACN,KAAK,EACL,kBAAkB,EAClB,GAAG,CAAC,aAAa,IAAI,IAAI,CAC1B,CAAC;YACF,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YACtE,WAAW,CAAC,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC;YAElF,MAAM,QAAQ,GAAG,MAAM,GAAG;iBACvB,IAAI,CACH;gBACE,QAAQ,EAAE,QAAQ;gBAClB,aAAa,EAAE,aAAa;gBAC5B,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;gBACrB,GAAG,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;aAC/E,EACD,EAAE,OAAO,EAAE,CACZ;iBACA,OAAO,EAAE,CAAC;YACb,MAAM,GAAG;gBACP,KAAK,EAAE,WAAW;gBAClB,UAAU,EAAE,QAAQ,CAAC,GAAG,CACtB,CAAC,GAAG,EAAE,EAAE,CACN,IAAI,WAAW,CACb,GAAG,CAAC,GAAG,EACP,cAAc,EACd,GAAG,CAAC,WAAW,IAAI,CAAC,EACpB,GAAG,CAAC,WAAW,EACf,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,EAClF,GAAG,CAAC,aAAa,IAAI,IAAI,CAC1B,CACJ;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO,MAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAqB,EAAE,OAA4B;QACxE,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1C,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzC,SAAS,CAAC;YACT;gBACE,MAAM,EAAE;oBACN,OAAO,EAAE,IAAI,CAAC,QAAQ;oBACtB,MAAM,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE;oBAC7B,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE;iBAClC;aACF;YACD;gBACE,KAAK,EAAE;oBACL,GAAG,EAAE,CAAC,CAAC;iBACR;aACF;YACD;gBACE,MAAM,EAAE;oBACN,GAAG,EAAE,MAAM;oBACX,iBAAiB,EAAE;wBACjB,MAAM,EAAE,oBAAoB;qBAC7B;iBACF;aACF;SACF,CAAC;aACD,OAAO,EAAE,CAAC;QACb,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACzC,OAAO,GAAG,CAAC,iBAAiB,CAAC;QAC/B,CAAC,CAAC,CAAC;QACH,OAAO,iBAAiB,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,CAAC,kBAAkB,CACvB,UAAqB,EACrB,WAAgC,EAChC,OAAgC;QAEhC,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,IAAI,OAAO,GAAuC,EAAE,CAAC;QAErD,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChE,KAAK,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE;oBACH,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,IAAI;wBACP,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;qBACjB;oBACD,IAAI,EAAE;wBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,IAAI;wBACP,CAAC,EAAE,GAAU;qBACd;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,4BAA4B,CAAC;QAC7D,MAAM,SAAS,GAAG,OAAO,EAAE,eAAe,IAAI,kCAAkC,CAAC;QAEjF,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CACrC;YACE,GAAG,EAAE,OAAO;SACb,EACD;YACE,OAAO,EAAE,SAAS;YAClB,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;YAChB,KAAK,EAAE,KAAK;YACZ,oFAAoF;YACpF,UAAU;YACV,SAAS,EAAE,KAAK;YAChB,sEAAsE;YACtE,qBAAqB;YACrB,8EAA8E;YAC9E,yDAAyD;YACzD,yCAAyC;YACzC,GAAG,EAAE,IAAI;YAET,gFAAgF;YAChF,gBAAgB;YAChB,qFAAqF;YACrF,qCAAqC;YACrC,WAAW,EAAE,KAAK;SACnB,CACqC,CAAC;QAEzC,yEAAyE;QACzE,iFAAiF;QACjF,mDAAmD;QACnD,+EAA+E;QAC/E,uBAAuB;QACvB,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;YACzB,0EAA0E;YAC1E,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,YAAY,GAA+B,IAAI,CAAC;QAEpD,uDAAuD;QACvD,KAAK,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,wBAAwB,CAAuB,CAAC;YACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAEzB,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,IAAI,MAAM,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;gBACpF,IAAI,KAAK,GAAuB,SAAS,CAAC;gBAC1C,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;oBACzB,IAAI,YAAY,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;wBAClC,YAAY,CAAC,QAAQ,GAAG,IAAI,CAAC;oBAC/B,CAAC;oBAED,MAAM,UAAU,GAAG,YAAY,CAAC;oBAChC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;oBAC3B,YAAY,GAAG,IAAI,CAAC;oBACpB,SAAS,GAAG,CAAC,CAAC;oBACd,MAAM,UAAU,CAAC;gBACnB,CAAC;gBAED,KAAK,KAAL,KAAK,GAAK,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAC;gBAClC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBACD,YAAY,GAAG;oBACb,MAAM;oBACN,KAAK,EAAE,KAAK;oBACZ,QAAQ,EAAE,OAAO;oBACjB,IAAI,EAAE,EAAE;oBACR,UAAU,EAAE,KAAK;iBAClB,CAAC;YACJ,CAAC;YAED,MAAM,KAAK,GAAoB;gBAC7B,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtC,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,WAAW,EAAE,GAAG,CAAC,KAAK;gBACtB,SAAS,EAAE,GAAG,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE;gBAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;aACf,CAAC;YACF,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,YAAY,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;YAEtC,SAAS,IAAI,OAAO,CAAC,UAAU,CAAC;QAClC,CAAC;QAED,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,YAAY,CAAC;YAChC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,UAAU,CAAC;QACnB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAqB,EAAE,OAAiB;QACzD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,OAAO,GAAU,EAAE,CAAC;QAC1B,KAAK,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE;oBACH,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,IAAI;wBACP,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE;qBACrB;oBACD,IAAI,EAAE;wBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,IAAI;wBACP,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;qBACtB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aACxC,SAAS,CACR;YACE;gBACE,MAAM,EAAE;oBACN,GAAG,EAAE,OAAO;iBACb;aACF;YACD;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;aACrF;SACF,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,CACvB;aACA,OAAO,EAAE,CAAC;QAEb,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC3B,OAAO;gBACL,MAAM,EAAE,GAAG,CAAC,GAAG;gBACf,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU;aACxE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEnB,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,KAAK,EAAE,aAAa,CAAC,UAAU;gBAC/B,aAAa,EAAE,IAAI;gBACnB,aAAa,EAAE,KAAK;aACrB;SACF,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,UAAU,EAAE;gBACV,aAAa,EAAE,CAAC;gBAChB,mBAAmB,EAAE,CAAC;gBACtB,KAAK,EAAE,CAAC;aACT;SACF,CACF,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,QAAQ;YAC7B,cAAc,EAAE,GAAG,CAAC,mBAAmB;SACxC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,qEAAqE;QACrE,uEAAuE;QAEvE,uBAAuB;QACvB,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,KAAK;gBACpB,aAAa,EAAE,IAAI;gBACnB,mBAAmB,EAAE,IAAI;gBACzB,eAAe,EAAE,IAAI;gBACrB,oBAAoB,EAAE,IAAI;aAC3B;SACF,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QACF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;YACE,GAAG,EAAE,cAAc,CAAgB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACrE,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QACF,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CACxC;YACE,GAAG,EAAE,cAAc,CAAY,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACjE,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QAEF,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,CACnC;YACE,GAAG,EAAE,cAAc,CAAY,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACjE,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QAEF,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,UAAU,CACpC;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,GAAW;QAC/B,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,IAAI;gBACnB,aAAa,EAAE,GAAG;gBAClB,kBAAkB,EAAE,IAAI,IAAI,EAAE;aAC/B;SACF,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;gBACvC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClF,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,IAAI,YAAY,EAAE,CAAC;oBACrC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;wBACE,GAAG,EAAE,IAAI,CAAC,QAAQ;qBACnB,EACD;wBACE,IAAI,EAAE;4BACJ,KAAK,EAAE,aAAa,CAAC,MAAM;yBAC5B;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;wBACE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE;wBAC3B,KAAK,EAAE,aAAa,CAAC,MAAM;qBAC5B,EACD;wBACE,IAAI,EAAE;4BACJ,KAAK,EAAE,aAAa,CAAC,IAAI;yBAC1B;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,CAAM;QACtB,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,qBAAqB,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,gBAAgB,EAAE,OAAO;aAC1B;SACF,CACF,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"MongoSyncBucketStorage.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoSyncBucketStorage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,WAAW,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAGL,4BAA4B,EAC5B,kCAAkC,EAMnC,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAgD,aAAa,EAAE,MAAM,aAAa,CAAC;AAC1F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACvG,OAAO,EAAE,aAAa,EAA8B,MAAM,qBAAqB,CAAC;AAEhF,MAAM,OAAO,sBAAsB;IAQjC,YACkB,OAA2B,EAC3B,QAAgB,EAChB,UAAwB,EACxB,SAAiB;QAHjB,YAAO,GAAP,OAAO,CAAoB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,eAAU,GAAV,UAAU,CAAc;QACxB,cAAS,GAAT,SAAS,CAAQ;QAV3B,kBAAa,GAAG,IAAI,aAAa,CAAC;YACxC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;SACF,CAAC,CAAC;QAQD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EACtB;YACE,UAAU,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE;SAC3D,CACF,CAAC;QACF,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,eAAe,IAAI,EAAE,CAAC;YAC5D,GAAG,EAAE,GAAG,EAAE,mBAAmB,IAAI,WAAW,CAAC,QAAQ;SACtD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAW,EAAE,QAAsD;QAClF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD,EAAE,UAAU,EAAE,EAAE,mBAAmB,EAAE,CAAC,EAAE,oBAAoB,EAAE,CAAC,EAAE,EAAE,CACpE,CAAC;QACF,MAAM,cAAc,GAAG,GAAG,EAAE,mBAAmB,IAAI,IAAI,CAAC;QAExD,MAAM,KAAK,GAAG,IAAI,gBAAgB,CAChC,IAAI,CAAC,EAAE,EACP,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,SAAS,EACd,cAAc,EACd,GAAG,EAAE,oBAAoB,IAAI,IAAI,CAClC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC7C,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAEtE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,GAAG,QAAQ,CAAC;QAEzE,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACtG,IAAI,MAAM,GAA8B,IAAI,CAAC;QAC7C,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC;YAClC,IAAI,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CACzB;gBACE,QAAQ,EAAE,QAAQ;gBAClB,aAAa,EAAE,aAAa;gBAC5B,WAAW,EAAE,UAAU;gBACvB,WAAW,EAAE,MAAM;gBACnB,UAAU,EAAE,KAAK;gBACjB,mBAAmB,EAAE,OAAO;aAC7B,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;YACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,GAAG,GAAG;oBACJ,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE;oBACxB,QAAQ,EAAE,QAAQ;oBAClB,aAAa,EAAE,aAAa;oBAC5B,WAAW,EAAE,UAAU;oBACvB,WAAW,EAAE,MAAM;oBACnB,UAAU,EAAE,KAAK;oBACjB,kBAAkB,EAAE,IAAI;oBACxB,mBAAmB,EAAE,OAAO;oBAC5B,aAAa,EAAE,KAAK;iBACrB,CAAC;gBAEF,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACxC,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,GAAG,CAAC,GAAG,EACP,cAAc,EACd,UAAU,EACV,MAAM,EACN,KAAK,EACL,kBAAkB,EAClB,GAAG,CAAC,aAAa,IAAI,IAAI,CAC1B,CAAC;YACF,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YACtE,WAAW,CAAC,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC;YAElF,MAAM,QAAQ,GAAG,MAAM,GAAG;iBACvB,IAAI,CACH;gBACE,QAAQ,EAAE,QAAQ;gBAClB,aAAa,EAAE,aAAa;gBAC5B,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;gBACrB,GAAG,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;aAC/E,EACD,EAAE,OAAO,EAAE,CACZ;iBACA,OAAO,EAAE,CAAC;YACb,MAAM,GAAG;gBACP,KAAK,EAAE,WAAW;gBAClB,UAAU,EAAE,QAAQ,CAAC,GAAG,CACtB,CAAC,GAAG,EAAE,EAAE,CACN,IAAI,WAAW,CACb,GAAG,CAAC,GAAG,EACP,cAAc,EACd,GAAG,CAAC,WAAW,IAAI,CAAC,EACpB,GAAG,CAAC,WAAW,EACf,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,EAClF,GAAG,CAAC,aAAa,IAAI,IAAI,CAC1B,CACJ;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO,MAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAqB,EAAE,OAA4B;QACxE,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1C,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzC,SAAS,CAAC;YACT;gBACE,MAAM,EAAE;oBACN,OAAO,EAAE,IAAI,CAAC,QAAQ;oBACtB,MAAM,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE;oBAC7B,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE;iBAClC;aACF;YACD;gBACE,KAAK,EAAE;oBACL,GAAG,EAAE,CAAC,CAAC;iBACR;aACF;YACD;gBACE,MAAM,EAAE;oBACN,GAAG,EAAE,MAAM;oBACX,iBAAiB,EAAE;wBACjB,MAAM,EAAE,oBAAoB;qBAC7B;iBACF;aACF;SACF,CAAC;aACD,OAAO,EAAE,CAAC;QACb,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACzC,OAAO,GAAG,CAAC,iBAAiB,CAAC;QAC/B,CAAC,CAAC,CAAC;QACH,OAAO,iBAAiB,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,CAAC,kBAAkB,CACvB,UAAqB,EACrB,WAAgC,EAChC,OAAgC;QAEhC,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,IAAI,OAAO,GAAuC,EAAE,CAAC;QAErD,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChE,KAAK,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE;oBACH,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,IAAI;wBACP,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;qBACjB;oBACD,IAAI,EAAE;wBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,IAAI;wBACP,CAAC,EAAE,GAAU;qBACd;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,4BAA4B,CAAC;QAC7D,MAAM,SAAS,GAAG,OAAO,EAAE,eAAe,IAAI,kCAAkC,CAAC;QAEjF,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CACrC;YACE,GAAG,EAAE,OAAO;SACb,EACD;YACE,OAAO,EAAE,SAAS;YAClB,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;YAChB,KAAK,EAAE,KAAK;YACZ,oFAAoF;YACpF,UAAU;YACV,SAAS,EAAE,KAAK;YAChB,sEAAsE;YACtE,qBAAqB;YACrB,8EAA8E;YAC9E,yDAAyD;YACzD,yCAAyC;YACzC,GAAG,EAAE,IAAI;YAET,gFAAgF;YAChF,gBAAgB;YAChB,qFAAqF;YACrF,qCAAqC;YACrC,WAAW,EAAE,KAAK;SACnB,CACqC,CAAC;QAEzC,yEAAyE;QACzE,iFAAiF;QACjF,mDAAmD;QACnD,+EAA+E;QAC/E,uBAAuB;QACvB,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;YACzB,0EAA0E;YAC1E,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,YAAY,GAA+B,IAAI,CAAC;QAEpD,uDAAuD;QACvD,KAAK,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,wBAAwB,CAAuB,CAAC;YACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAEzB,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,IAAI,MAAM,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;gBACpF,IAAI,KAAK,GAAuB,SAAS,CAAC;gBAC1C,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;oBACzB,IAAI,YAAY,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;wBAClC,YAAY,CAAC,QAAQ,GAAG,IAAI,CAAC;oBAC/B,CAAC;oBAED,MAAM,UAAU,GAAG,YAAY,CAAC;oBAChC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;oBAC3B,YAAY,GAAG,IAAI,CAAC;oBACpB,SAAS,GAAG,CAAC,CAAC;oBACd,MAAM,UAAU,CAAC;gBACnB,CAAC;gBAED,KAAK,KAAL,KAAK,GAAK,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAC;gBAClC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBACD,YAAY,GAAG;oBACb,MAAM;oBACN,KAAK,EAAE,KAAK;oBACZ,QAAQ,EAAE,OAAO;oBACjB,IAAI,EAAE,EAAE;oBACR,UAAU,EAAE,KAAK;iBAClB,CAAC;YACJ,CAAC;YAED,MAAM,KAAK,GAAoB;gBAC7B,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtC,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,WAAW,EAAE,GAAG,CAAC,KAAK;gBACtB,SAAS,EAAE,GAAG,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE;gBAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;aACf,CAAC;YACF,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,YAAY,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;YAEtC,SAAS,IAAI,OAAO,CAAC,UAAU,CAAC;QAClC,CAAC;QAED,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,YAAY,CAAC;YAChC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,UAAU,CAAC;QACnB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAqB,EAAE,OAAiB;QACzD,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,KAAmC;QACpE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,OAAO,GAAU,EAAE,CAAC;QAC1B,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE;oBACH,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;wBACjB,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE;qBAC7D;oBACD,IAAI,EAAE;wBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;wBACjB,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;qBACvB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aACxC,SAAS,CACR;YACE;gBACE,MAAM,EAAE;oBACN,GAAG,EAAE,OAAO;iBACb;aACF;YACD;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;aACrF;SACF,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,CACvB;aACA,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,GAAG,CACZ,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACpB,OAAO;gBACL,GAAG,CAAC,GAAG;gBACP;oBACE,MAAM,EAAE,GAAG,CAAC,GAAG;oBACf,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU;iBAC1C;aAChC,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEnB,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,KAAK,EAAE,aAAa,CAAC,UAAU;gBAC/B,aAAa,EAAE,IAAI;gBACnB,aAAa,EAAE,KAAK;aACrB;SACF,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,UAAU,EAAE;gBACV,aAAa,EAAE,CAAC;gBAChB,mBAAmB,EAAE,CAAC;gBACtB,KAAK,EAAE,CAAC;aACT;SACF,CACF,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,QAAQ;YAC7B,cAAc,EAAE,GAAG,CAAC,mBAAmB;SACxC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,qEAAqE;QACrE,uEAAuE;QAEvE,uBAAuB;QACvB,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,KAAK;gBACpB,aAAa,EAAE,IAAI;gBACnB,mBAAmB,EAAE,IAAI;gBACzB,eAAe,EAAE,IAAI;gBACrB,oBAAoB,EAAE,IAAI;aAC3B;SACF,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QACF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;YACE,GAAG,EAAE,cAAc,CAAgB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACrE,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QACF,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CACxC;YACE,GAAG,EAAE,cAAc,CAAY,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACjE,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QAEF,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,CACnC;YACE,GAAG,EAAE,cAAc,CAAY,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACjE,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;QAEF,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,UAAU,CACpC;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,0BAA0B,EAAE,CACnD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,GAAW;QAC/B,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,IAAI;gBACnB,aAAa,EAAE,GAAG;gBAClB,kBAAkB,EAAE,IAAI,IAAI,EAAE;aAC/B;SACF,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;gBACvC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClF,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,IAAI,YAAY,EAAE,CAAC;oBACrC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;wBACE,GAAG,EAAE,IAAI,CAAC,QAAQ;qBACnB,EACD;wBACE,IAAI,EAAE;4BACJ,KAAK,EAAE,aAAa,CAAC,MAAM;yBAC5B;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;wBACE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE;wBAC3B,KAAK,EAAE,aAAa,CAAC,MAAM;qBAC5B,EACD;wBACE,IAAI,EAAE;4BACJ,KAAK,EAAE,aAAa,CAAC,IAAI;yBAC1B;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,CAAM;QACtB,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,qBAAqB,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,gBAAgB,EAAE,OAAO;aAC1B;SACF,CACF,CAAC;IACJ,CAAC;CACF"}

package/dist/sync/sync.js

@@ -48,8 +48,8 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
     // Bucket state of bucket id -> op_id.
     // This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
     let dataBuckets = new Map();
-    let last_checksums = null;
-    let last_write_checkpoint = null;
+    let lastChecksums = null;
+    let lastWriteCheckpoint = null;
     const { raw_data, binary_data } = params;
     if (params.buckets) {
         for (let { name, after: start } of params.buckets) {
@@ -76,31 +76,35 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
             // TODO: Limit number of buckets even before we get to this point
             throw new Error(`Too many buckets: ${allBuckets.length}`);
         }
-        let checksums = undefined;
         let dataBucketsNew = new Map();
         for (let bucket of allBuckets) {
             dataBucketsNew.set(bucket, dataBuckets.get(bucket) ?? '0');
         }
         dataBuckets = dataBucketsNew;
-        checksums = await storage.getChecksums(checkpoint, [...dataBuckets.keys()]);
-        if (last_checksums) {
-            const diff = util.checksumsDiff(last_checksums, checksums);
-            if (last_write_checkpoint == writeCheckpoint &&
-                diff.removed_buckets.length == 0 &&
-                diff.updated_buckets.length == 0) {
+        const bucketList = [...dataBuckets.keys()];
+        const checksumMap = await storage.getChecksums(checkpoint, bucketList);
+        // Subset of buckets for which there may be new data in this batch.
+        let bucketsToFetch;
+        if (lastChecksums) {
+            const diff = util.checksumsDiff(lastChecksums, checksumMap);
+            if (lastWriteCheckpoint == writeCheckpoint &&
+                diff.removedBuckets.length == 0 &&
+                diff.updatedBuckets.length == 0) {
                 // No changes - don't send anything to the client
                 continue;
             }
+            bucketsToFetch = diff.updatedBuckets.map((c) => c.bucket);
             let message = `Updated checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
             message += `buckets: ${allBuckets.length} | `;
-            message += `updated: ${limitedBuckets(diff.updated_buckets, 20)} | `;
-            message += `removed: ${limitedBuckets(diff.removed_buckets, 20)} | `;
+            message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
+            message += `removed: ${limitedBuckets(diff.removedBuckets, 20)} | `;
             micro.logger.info(message);
             const checksum_line = {
                 checkpoint_diff: {
                     last_op_id: checkpoint,
                     write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
-                    ...diff
+                    removed_buckets: diff.removedBuckets,
+                    updated_buckets: diff.updatedBuckets
                 }
             };
             yield checksum_line;
@@ -109,24 +113,27 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
             let message = `New checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
             message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
             micro.logger.info(message);
+            bucketsToFetch = allBuckets;
             const checksum_line = {
                 checkpoint: {
                     last_op_id: checkpoint,
                     write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
-                    buckets: checksums
+                    buckets: [...checksumMap.values()]
                 }
             };
             yield checksum_line;
         }
-        last_checksums = checksums;
-        last_write_checkpoint = writeCheckpoint;
-        yield* bucketDataInBatches(storage, checkpoint, dataBuckets, raw_data, binary_data, signal);
+        lastChecksums = checksumMap;
+        lastWriteCheckpoint = writeCheckpoint;
+        // This incrementally updates dataBuckets with each individual bucket position.
+        // At the end of this, we can be sure that all buckets have data up to the checkpoint.
+        yield* bucketDataInBatches({ storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal });
         await new Promise((resolve) => setTimeout(resolve, 10));
     }
 }
-async function* bucketDataInBatches(storage, checkpoint, dataBuckets, raw_data, binary_data, signal) {
+async function* bucketDataInBatches(request) {
     let isDone = false;
-    while (!signal.aborted && !isDone) {
+    while (!request.signal.aborted && !isDone) {
         // The code below is functionally the same as this for-await loop below.
         // However, the for-await loop appears to have a memory leak, so we avoid it.
         // for await (const { done, data } of bucketDataBatch(storage, checkpoint, dataBuckets, raw_data, signal)) {
@@ -136,7 +143,7 @@ async function* bucketDataInBatches(storage, checkpoint, dataBuckets, raw_data,
         //   }
         //   break;
         // }
-        const iter = bucketDataBatch(storage, checkpoint, dataBuckets, raw_data, binary_data, signal);
+        const iter = bucketDataBatch(request);
         try {
             while (true) {
                 const { value, done: iterDone } = await iter.next();
@@ -160,10 +167,14 @@ async function* bucketDataInBatches(storage, checkpoint, dataBuckets, raw_data,
 /**
  * Extracted as a separate internal function just to avoid memory leaks.
  */
-async function* bucketDataBatch(storage, checkpoint, dataBuckets, raw_data, binary_data, signal) {
+async function* bucketDataBatch(request) {
+    const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal } = request;
     const [_, release] = await syncSemaphore.acquire();
     try {
-        const data = storage.getBucketDataBatch(checkpoint, dataBuckets);
+        // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
+        // For the first batch, this will be all buckets.
+        const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)]));
+        const data = storage.getBucketDataBatch(checkpoint, filteredBuckets);
         let has_more = false;
         for await (let r of data) {
             if (signal.aborted) {