@powersync/service-module-mongodb-storage 0.15.4 → 0.16.0

package/dist/storage/implementation/v1/MongoCompactorV1.js

@@ -0,0 +1,52 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
+import { MongoCompactor } from '../MongoCompactor.js';
+import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
+export class MongoCompactorV1 extends MongoCompactor {
+    async *dirtyBucketBatches(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes.
+        // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is
+        // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting.
+        yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() }, { g: this.group_id, b: new mongo.MaxKey() }, options, () => null);
+    }
+    async dirtyBucketBatchForChecksums(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets
+        // estimate_since_compact as it progresses.
+        return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketStateV1, {
+            '_id.g': this.group_id,
+            'estimate_since_compact.count': { $gte: options.minBucketChanges }
+        }, () => null);
+    }
+    async writeBucketStateUpdates() {
+        await this.db.bucketStateV1.bulkWrite(this.bucketStateUpdates, { ordered: false });
+    }
+    async computeChecksumsForBuckets(buckets) {
+        return this.storage.checksums.computePartialChecksumsDirectByBucket(buckets.map(({ bucket }) => ({
+            bucket,
+            end: this.maxOpId
+        })));
+    }
+    bucketStateFilter(bucket, _definitionId) {
+        return {
+            _id: {
+                g: this.group_id,
+                b: bucket
+            }
+        };
+    }
+    async getBucketDataContext(bucket, _definitionId) {
+        return new SingleBucketStoreV1(this.db, {
+            replicationStreamId: this.group_id,
+            definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
+            bucket
+        });
+    }
+}
+//# sourceMappingURL=MongoCompactorV1.js.map

package/dist/storage/implementation/v1/MongoCompactorV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAI9E,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAe,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK3C,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,MAAc,EACd,aAAwC;QAExC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts

@@ -0,0 +1,9 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoParameterCompactorV1 extends MongoParameterCompactor {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+    protected collectionFilter(): mongo.Document;
+    protected deleteFilter(doc: mongo.Document): mongo.Document;
+}

package/dist/storage/implementation/v1/MongoParameterCompactorV1.js

@@ -0,0 +1,20 @@
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+export class MongoParameterCompactorV1 extends MongoParameterCompactor {
+    async getCollections() {
+        return [this.db.parameterIndexV1];
+    }
+    collectionFilter() {
+        return {
+            'key.g': this.group_id
+        };
+    }
+    deleteFilter(doc) {
+        return {
+            'key.g': doc.key.g,
+            lookup: doc.lookup,
+            _id: { $lte: doc._id },
+            key: doc.key
+        };
+    }
+}
+//# sourceMappingURL=MongoParameterCompactorV1.js.map

package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoParameterCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoParameterCompactorV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAGxE,MAAM,OAAO,yBAA0B,SAAQ,uBAAuB;IAG1D,KAAK,CAAC,cAAc;QAC5B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,gBAA+D,CAAC,CAAC;IACnF,CAAC;IAES,gBAAgB;QACxB,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC;IACJ,CAAC;IAES,YAAY,CAAC,GAAmB;QACxC,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAW;YAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG,EAAE,GAAG,CAAC,GAAG;SACb,CAAC;IACJ,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts

@@ -0,0 +1,41 @@
+import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage, utils } from '@powersync/service-core';
+import { ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import { MongoBucketStorage } from '../../MongoBucketStorage.js';
+import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js';
+import { CommonSourceTableDocument } from '../models.js';
+import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { MongoChecksums } from '../MongoChecksums.js';
+import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
+import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
+    readonly db: VersionedPowerSyncMongoV1;
+    readonly checksums: MongoChecksumsV1;
+    constructor(factory: MongoBucketStorage, group_id: number, sync_rules: MongoPersistedSyncRulesContent, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
+    protected initializeVersionStorage(): Promise<void>;
+    protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch;
+    protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
+    createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
+    protected createMongoParameterCompactor(checkpoint: InternalOpId, options: storage.CompactOptions): MongoParameterCompactor;
+    protected sourceTableBaseId(): Partial<CommonSourceTableDocument>;
+    protected augmentCreatedSourceTableDocument(_createDoc: CommonSourceTableDocument, _options: storage.ResolveTableOptions, _candidateSourceTable: storage.SourceTable): void;
+    protected initializeResolvedSourceRecords(_sourceTableId: bson.ObjectId): Promise<void>;
+    protected get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>;
+    protected getParameterSetsImpl(checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
+    protected getBucketDataBatchImpl(checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
+    protected clearBucketData(signal?: AbortSignal): Promise<void>;
+    protected clearParameterIndexes(signal?: AbortSignal): Promise<void>;
+    protected clearSourceRecords(signal?: AbortSignal): Promise<void>;
+    protected clearBucketState(signal?: AbortSignal): Promise<void>;
+    protected clearSourceTables(signal?: AbortSignal): Promise<void>;
+    protected getDataBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
+    protected getParameterBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
+}
+export declare function getParameterSetsV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
+export declare function getBucketDataBatchV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
+export declare function getDataBucketChangesV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
+export declare function getParameterBucketChangesV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;

package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js

@@ -0,0 +1,283 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { deserializeParameterLookup, internalToExternalOpId, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
+import { JSONBig } from '@powersync/service-jsonbig';
+import * as bson from 'bson';
+import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
+import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js';
+import { loadBucketDataDocumentV1 } from './models.js';
+import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js';
+import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
+import { MongoCompactorV1 } from './MongoCompactorV1.js';
+import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js';
+export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
+    constructor(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options) {
+        super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+    }
+    async initializeVersionStorage() { }
+    createWriterImpl(batchOptions) {
+        return new MongoBucketBatchV1(batchOptions);
+    }
+    createMongoChecksums(options) {
+        return new MongoChecksumsV1(this.db, this.group_id, {
+            ...options.checksumOptions,
+            storageConfig: options?.storageConfig,
+            mapping: this.sync_rules.mapping
+        });
+    }
+    createMongoCompactor(options) {
+        return new MongoCompactorV1(this, this.db, options);
+    }
+    createMongoParameterCompactor(checkpoint, options) {
+        return new MongoParameterCompactorV1(this.db, this.group_id, checkpoint, options);
+    }
+    sourceTableBaseId() {
+        return { group_id: this.group_id };
+    }
+    augmentCreatedSourceTableDocument(_createDoc, _options, _candidateSourceTable) { }
+    async initializeResolvedSourceRecords(_sourceTableId) { }
+    get versionContext() {
+        return {
+            db: this.db,
+            group_id: this.group_id,
+            mapping: this.mapping
+        };
+    }
+    getParameterSetsImpl(checkpoint, lookups, limit) {
+        return getParameterSetsV1(this.versionContext, checkpoint, lookups, limit);
+    }
+    getBucketDataBatchImpl(checkpoint, dataBuckets, options) {
+        return getBucketDataBatchV1(this.versionContext, checkpoint, dataBuckets, options);
+    }
+    async clearBucketData(signal) {
+        await this.clearDeleteMany('bucket data', () => this.db.bucket_data.deleteMany({
+            _id: idPrefixFilter({ g: this.group_id }, ['b', 'o'])
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
+    }
+    async clearParameterIndexes(signal) {
+        await this.clearDeleteMany('parameter index', () => this.db.parameterIndexV1.deleteMany({
+            'key.g': this.group_id
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
+    }
+    async clearSourceRecords(signal) {
+        await this.clearDeleteMany('source records', () => this.db.sourceRecordsV1.deleteMany({
+            _id: idPrefixFilter({ g: this.group_id }, ['t', 'k'])
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
+    }
+    async clearBucketState(signal) {
+        await this.clearDeleteMany('bucket state', () => this.db.bucketStateV1.deleteMany({
+            _id: idPrefixFilter({ g: this.group_id }, ['b'])
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
+    }
+    async clearSourceTables(signal) {
+        await this.clearDeleteMany('source tables', () => this.db.commonSourceTables(this.group_id).deleteMany({
+            group_id: this.group_id
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
+    }
+    getDataBucketChangesImpl(options) {
+        return getDataBucketChangesV1(this.versionContext, options);
+    }
+    getParameterBucketChangesImpl(options) {
+        return getParameterBucketChangesV1(this.versionContext, options);
+    }
+}
+export async function getParameterSetsV1(ctx, checkpoint, lookups, limit) {
+    return ctx.db.client.withSession({ snapshot: true }, async (session) => {
+        setSessionSnapshotTime(session, checkpoint.snapshotTime);
+        const lookupFilter = lookups.map((lookup) => {
+            return storage.serializeLookup(lookup);
+        });
+        const rows = await ctx.db.parameterIndexV1
+            .aggregate([
+            {
+                $match: {
+                    'key.g': ctx.group_id,
+                    lookup: { $in: lookupFilter },
+                    _id: { $lte: checkpoint.checkpoint }
+                }
+            },
+            {
+                $set: {
+                    index: { $indexOfArray: [lookupFilter, '$lookup'] }
+                }
+            },
+            {
+                $sort: {
+                    _id: -1
+                }
+            },
+            {
+                $group: {
+                    _id: { key: '$key', lookup: '$index' },
+                    bucket_parameters: {
+                        $first: '$bucket_parameters'
+                    }
+                }
+            },
+            {
+                $project: {
+                    _id: false,
+                    lookup: '$_id.lookup',
+                    bucket_parameters: true
+                }
+            },
+            { $unwind: '$bucket_parameters' },
+            {
+                $limit: limit + 1
+            }
+        ], {
+            session,
+            readConcern: 'snapshot',
+            maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
+        })
+            .toArray()
+            .catch((e) => {
+            throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries');
+        });
+        if (rows.length > limit) {
+            throw new ParameterSetLimitExceededError(limit);
+        }
+        const byLookup = Map.groupBy(rows, (row) => lookups[row.lookup]);
+        const results = [];
+        byLookup.forEach((value, lookup) => results.push({ lookup, rows: value.map((r) => r.bucket_parameters) }));
+        return results;
+    });
+}
+export async function* getBucketDataBatchV1(ctx, checkpoint, dataBuckets, options) {
+    if (dataBuckets.length == 0) {
+        return;
+    }
+    let filters = [];
+    const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
+    if (checkpoint == null) {
+        throw new Error('checkpoint is null');
+    }
+    const end = checkpoint;
+    for (let { bucket: name, start } of dataBuckets) {
+        filters.push({
+            _id: {
+                $gt: {
+                    g: ctx.group_id,
+                    b: name,
+                    o: start
+                },
+                $lte: {
+                    g: ctx.group_id,
+                    b: name,
+                    o: end
+                }
+            }
+        });
+    }
+    const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
+    const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
+    const cursor = ctx.db.bucket_data.find({
+        $or: filters
+    }, {
+        session: undefined,
+        sort: { _id: 1 },
+        limit: batchLimit,
+        batchSize: batchLimit + 1,
+        raw: true,
+        maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
+    });
+    let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => {
+        throw lib_mongo.mapQueryError(e, 'while reading bucket data');
+    });
+    if (data.length == batchLimit) {
+        batchHasMore = true;
+    }
+    let chunkSizeBytes = 0;
+    let currentChunk = null;
+    let targetOp = null;
+    for (let rawData of data) {
+        const row = loadBucketDataDocumentV1(bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS));
+        const bucket = row.bucketKey.bucket;
+        if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) {
+            let start = undefined;
+            if (currentChunk != null) {
+                if (currentChunk.bucket == bucket) {
+                    currentChunk.has_more = true;
+                    start = currentChunk.next_after;
+                }
+                const yieldChunk = currentChunk;
+                currentChunk = null;
+                chunkSizeBytes = 0;
+                yield { chunkData: yieldChunk, targetOp: targetOp };
+                targetOp = null;
+            }
+            if (start == null) {
+                const startOpId = bucketMap.get(bucket);
+                if (startOpId == null) {
+                    throw new Error(`data for unexpected bucket: ${bucket}`);
+                }
+                start = internalToExternalOpId(startOpId);
+            }
+            currentChunk = {
+                bucket,
+                after: start,
+                has_more: false,
+                data: [],
+                next_after: start
+            };
+            targetOp = null;
+        }
+        const entry = mapOpEntry(row);
+        if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) {
+            targetOp = row.target_op;
+        }
+        currentChunk.data.push(entry);
+        currentChunk.next_after = entry.op_id;
+        chunkSizeBytes += rawData.byteLength;
+    }
+    if (currentChunk != null) {
+        const yieldChunk = currentChunk;
+        yieldChunk.has_more = batchHasMore;
+        yield { chunkData: yieldChunk, targetOp: targetOp };
+    }
+}
+export async function getDataBucketChangesV1(ctx, options) {
+    const limit = 1000;
+    const bucketStateUpdates = await ctx.db.bucketStateV1
+        .find({
+        '_id.g': ctx.group_id,
+        last_op: { $gt: options.lastCheckpoint.checkpoint }
+    }, {
+        projection: {
+            '_id.b': 1
+        },
+        limit: limit + 1,
+        batchSize: limit + 2,
+        singleBatch: true
+    })
+        .toArray();
+    const buckets = bucketStateUpdates.map((doc) => doc._id.b);
+    const invalidateDataBuckets = buckets.length > limit;
+    return {
+        invalidateDataBuckets,
+        updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets)
+    };
+}
+export async function getParameterBucketChangesV1(ctx, options) {
+    const limit = 1000;
+    const parameterUpdates = await ctx.db.parameterIndexV1
+        .find({
+        _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
+        'key.g': ctx.group_id
+    }, {
+        projection: {
+            lookup: 1
+        },
+        limit: limit + 1,
+        batchSize: limit + 2,
+        singleBatch: true
+    })
+        .toArray();
+    const invalidateParameterUpdates = parameterUpdates.length > limit;
+    return {
+        invalidateParameterBuckets: invalidateParameterUpdates,
+        updatedParameterLookups: invalidateParameterUpdates
+            ? new Set()
+            : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup))))
+    };
+}
+//# sourceMappingURL=MongoSyncBucketStorageV1.js.map

package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoSyncBucketStorageV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoSyncBucketStorageV1.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAEL,0BAA0B,EAG1B,sBAAsB,EACtB,8BAA8B,EAE9B,OAAO,EAER,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAErD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAY7G,OAAO,EAAE,sBAAsB,EAAiC,MAAM,8BAA8B,CAAC;AACrG,OAAO,EAA8D,wBAAwB,EAAE,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAG3E,MAAM,OAAO,wBAAyB,SAAQ,sBAAsB;IAKlE,YACE,OAA2B,EAC3B,QAAgB,EAChB,UAA0C,EAC1C,SAAiB,EACjB,mBAA4D,EAC5D,OAAsC;QAEtC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;IAChF,CAAC;IAES,KAAK,CAAC,wBAAwB,KAAmB,CAAC;IAElD,gBAAgB,CAAC,YAAqC;QAC9D,OAAO,IAAI,kBAAkB,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAES,oBAAoB,CAAC,OAAsC;QACnE,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE;YAClD,GAAG,OAAO,CAAC,eAAe;YAC1B,aAAa,EAAE,OAAO,EAAE,aAAa;YACrC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO;SACjC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAAC,OAA4B;QAC/C,OAAO,IAAI,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IAES,6BAA6B,CACrC,UAAwB,EACxB,OAA+B;QAE/B,OAAO,IAAI,yBAAyB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAES,iBAAiB;QACzB,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;IACrC,CAAC;IAES,iCAAiC,CACzC,UAAqC,EACrC,QAAqC,EACrC,qBAA0C,IACnC,CAAC;IAEA,KAAK,CAAC,+BAA+B,CAAC,cAA6B,IAAkB,CAAC;IAEhG,IAAuB,cAAc;QACnC,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IAES,oBAAoB,CAC5B,UAA4C,EAC5C,OAAgC,EAChC,KAAa;QAEb,OAAO,kBAAkB,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAC7E,CAAC;IAES,sBAAsB,CAC9B,UAA8B,EAC9B,WAAwC,EACxC,OAAwC;QAExC,OAAO,oBAAoB,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IAES,KAAK,CAAC,eAAe,CAAC,MAAoB;QAClD,MAAM,IAAI,CAAC,eAAe,CACxB,aAAa,EACb,GAAG,EAAE,CACH,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAC5B;YACE,GAAG,EAAE,cAAc,CAAkB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACvE,EACD,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,EACH,MAAM,CACP,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,qBAAqB,CAAC,MAAoB;QACxD,MAAM,IAAI,CAAC,eAAe,CACxB,iBAAiB,EACjB,GAAG,EAAE,CACH,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,UAAU,CACjC;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,EACD,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,EACH,MAAM,CACP,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,MAAoB;QACrD,MAAM,IAAI,CAAC,eAAe,CACxB,gBAAgB,EAChB,GAAG,EAAE,CACH,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,UAAU,CAChC;YACE,GAAG,EAAE,cAAc,CAAY,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACjE,EACD,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,EACH,MAAM,CACP,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,MAAoB;QACnD,MAAM,IAAI,CAAC,eAAe,CACxB,cAAc,EACd,GAAG,EAAE,CACH,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,UAAU,CAC9B;YACE,GAAG,EAAE,cAAc,CAA6B,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;SAC7E,EACD,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,EACH,MAAM,CACP,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,MAAoB;QACpD,MAAM,IAAI,CAAC,eAAe,CACxB,eAAe,EACf,GAAG,EAAE,CACH,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,UAAU,CAClD;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,EACH,MAAM,CACP,CAAC;IACJ,CAAC;IAES,wBAAwB,CAChC,OAAoC;QAEpC,OAAO,sBAAsB,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;IAES,6BAA6B,CACrC,OAAoC;QAEpC,OAAO,2BAA2B,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAA6D,EAC7D,UAA4C,EAC5C,OAAgC,EAChC,KAAa;IAEb,OAAO,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACrE,sBAAsB,CAAC,OAAO,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1C,OAAO,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,gBAAgB;aACvC,SAAS,CACR;YACE;gBACE,MAAM,EAAE;oBACN,OAAO,EAAE,GAAG,CAAC,QAAQ;oBACrB,MAAM,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE;oBAC7B,GAAG,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,UAAU,EAAE;iBACrC;aACF;YACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE,aAAa,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE;iBACpD;aACF;YACD;gBACE,KAAK,EAAE;oBACL,GAAG,EAAE,CAAC,CAAC;iBACR;aACF;YACD;gBACE,MAAM,EAAE;oBACN,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE;oBACtC,iBAAiB,EAAE;wBACjB,MAAM,EAAE,oBAAoB;qBAC7B;iBACF;aACF;YACD;gBACE,QAAQ,EAAE;oBACR,GAAG,EAAE,KAAK;oBACV,MAAM,EAAE,aAAa;oBACrB,iBAAiB,EAAE,IAAI;iBACxB;aACF;YACD,EAAE,OAAO,EAAE,oBAAoB,EAAE;YACjC;gBACE,MAAM,EAAE,KAAK,GAAG,CAAC;aAClB;SACF,EACD;YACE,OAAO;YACP,WAAW,EAAE,UAAU;YACvB,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,0BAA0B;SACnD,CACF;aACA,OAAO,EAAE;aACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;QAEL,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,8BAA8B,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAA0B,EAAE,CAAC;QAC1C,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3G,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,GAA6D,EAC7D,UAA8B,EAC9B,WAAwC,EACxC,OAAwC;IAExC,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IACD,IAAI,OAAO,GAAyC,EAAE,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEzF,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC;IACvB,KAAK,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,WAAW,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE;gBACH,GAAG,EAAE;oBACH,CAAC,EAAE,GAAG,CAAC,QAAQ;oBACf,CAAC,EAAE,IAAI;oBACP,CAAC,EAAE,KAAK;iBACT;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,GAAG,CAAC,QAAQ;oBACf,CAAC,EAAE,IAAI;oBACP,CAAC,EAAE,GAAU;iBACd;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,EAAE,KAAK,IAAI,OAAO,CAAC,4BAA4B,CAAC;IAC1E,MAAM,mBAAmB,GAAG,OAAO,EAAE,eAAe,IAAI,OAAO,CAAC,kCAAkC,CAAC;IAEnG,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CACpC;QACE,GAAG,EAAE,OAAO;KACb,EACD;QACE,OAAO,EAAE,SAAS;QAClB,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;QAChB,KAAK,EAAE,UAAU;QACjB,SAAS,EAAE,UAAU,GAAG,CAAC;QACzB,GAAG,EAAE,IAAI;QACT,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,0BAA0B;KACnD,CACqC,CAAC;IAEzC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9E,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;QAC9B,YAAY,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,YAAY,GAAgC,IAAI,CAAC;IACrD,IAAI,QAAQ,GAAwB,IAAI,CAAC;IAEzC,KAAK,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,wBAAwB,CAClC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,iCAAiC,CAAyB,CAC7F,CAAC;QACF,MAAM,MAAM,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC;QAEpC,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,IAAI,MAAM,IAAI,cAAc,IAAI,mBAAmB,EAAE,CAAC;YACnG,IAAI,KAAK,GAA6B,SAAS,CAAC;YAChD,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;gBACzB,IAAI,YAAY,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;oBAClC,YAAY,CAAC,QAAQ,GAAG,IAAI,CAAC;oBAC7B,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;gBAClC,CAAC;gBAED,MAAM,UAAU,GAAG,YAAY,CAAC;gBAChC,YAAY,GAAG,IAAI,CAAC;gBACpB,cAAc,GAAG,CAAC,CAAC;gBACnB,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;gBACpD,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YAED,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACxC,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;oBACtB,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBACD,KAAK,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;YAC5C,CAAC;YACD,YAAY,GAAG;gBACb,MAAM;gBACN,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,KAAK;gBACf,IAAI,EAAE,EAAE;gBACR,UAAU,EAAE,KAAK;aAClB,CAAC;YACF,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAE9B,IAAI,GAAG,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;YAC5E,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC;QAC3B,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,YAAY,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;QACtC,cAAc,IAAI,OAAO,CAAC,UAAU,CAAC;IACvC,CAAC;IAED,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,YAAY,CAAC;QAChC,UAAU,CAAC,QAAQ,GAAG,YAAY,CAAC;QACnC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,GAA6D,EAC7D,OAAoC;IAEpC,MAAM,KAAK,GAAG,IAAI,CAAC;IACnB,MAAM,kBAAkB,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,aAAa;SAClD,IAAI,CACH;QACE,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,OAAO,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE;KACpD,EACD;QACE,UAAU,EAAE;YACV,OAAO,EAAE,CAAC;SACX;QACD,KAAK,EAAE,KAAK,GAAG,CAAC;QAChB,SAAS,EAAE,KAAK,GAAG,CAAC;QACpB,WAAW,EAAE,IAAI;KAClB,CACF;SACA,OAAO,EAAE,CAAC;IAEb,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,qBAAqB,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAErD,OAAO;QACL,qBAAqB;QACrB,kBAAkB,EAAE,qBAAqB,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC;KACjF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,GAA6D,EAC7D,OAAoC;IAEpC,MAAM,KAAK,GAAG,IAAI,CAAC;IACnB,MAAM,gBAAgB,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,gBAAgB;SACnD,IAAI,CACH;QACE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE;QACxF,OAAO,EAAE,GAAG,CAAC,QAAQ;KACtB,EACD;QACE,UAAU,EAAE;YACV,MAAM,EAAE,CAAC;SACV;QACD,KAAK,EAAE,KAAK,GAAG,CAAC;QAChB,SAAS,EAAE,KAAK,GAAG,CAAC;QACpB,WAAW,EAAE,IAAI;KAClB,CACF;SACA,OAAO,EAAE,CAAC;IACb,MAAM,0BAA0B,GAAG,gBAAgB,CAAC,MAAM,GAAG,KAAK,CAAC;IAEnE,OAAO;QACL,0BAA0B,EAAE,0BAA0B;QACtD,uBAAuB,EAAE,0BAA0B;YACjD,CAAC,CAAC,IAAI,GAAG,EAAU;YACnB,CAAC,CAAC,IAAI,GAAG,CAAS,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;KAC1G,CAAC;AACJ,CAAC"}

package/dist/storage/implementation/v1/PersistedBatchV1.d.ts

@@ -0,0 +1,26 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { BucketDataSource } from '@powersync/service-sync-rules';
+import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
+import { PersistedBatch, SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+import { CurrentDataDocument } from './models.js';
+export declare class PersistedBatchV1 extends PersistedBatch {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[];
+    protected checkDefinitionId(_definitionId: BucketDefinitionId | null): BucketDefinitionId;
+    protected getBucketDefinitionId(_bucketSource: BucketDataSource): BucketDefinitionId;
+    saveParameterData(data: SaveParameterDataOptions): void;
+    hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void;
+    softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, _checkpointGreaterThan: bigint): void;
+    upsertCurrentData(values: UpsertCurrentDataOptions): void;
+    protected get currentDataCount(): number;
+    protected flushBucketData(session: mongo.ClientSession): Promise<void>;
+    protected flushBucketParameters(session: mongo.ClientSession): Promise<void>;
+    protected flushCurrentData(session: mongo.ClientSession): Promise<void>;
+    protected flushBucketStates(session: mongo.ClientSession): Promise<void>;
+    protected resetCurrentData(): void;
+    private getBucketStateUpdates;
+    private currentDataId;
+}

package/dist/storage/implementation/v1/PersistedBatchV1.js

@@ -0,0 +1,183 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import { mongoTableId } from '../../../utils/util.js';
+import { EMPTY_DATA } from '../MongoBucketBatchShared.js';
+import { PersistedBatch } from '../common/PersistedBatch.js';
+import { LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID } from '../models.js';
+import { serializeBucketDataV1, taggedBucketParameterDocumentToV1 } from './models.js';
+export class PersistedBatchV1 extends PersistedBatch {
+    currentData = [];
+    checkDefinitionId(_definitionId) {
+        // V1 storage doesn't persist the id, and we don't use it.
+        return LEGACY_BUCKET_DATA_DEFINITION_ID;
+    }
+    getBucketDefinitionId(_bucketSource) {
+        return LEGACY_BUCKET_DATA_DEFINITION_ID;
+    }
+    saveParameterData(data) {
+        const { sourceTable, sourceKey, evaluated } = data;
+        const remaining_lookups = new Map();
+        for (let lookup of data.existing_lookups) {
+            if (lookup.indexId != null) {
+                throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled');
+            }
+            remaining_lookups.set(lookup.lookup.toString('base64'), lookup.lookup);
+        }
+        for (let result of evaluated) {
+            const binLookup = storage.serializeLookup(result.lookup);
+            remaining_lookups.delete(binLookup.toString('base64'));
+            const op_id = data.op_seq.next();
+            this.debugLastOpId = op_id;
+            const values = {
+                _id: op_id,
+                key: {
+                    g: this.group_id,
+                    t: mongoTableId(sourceTable.id),
+                    k: sourceKey
+                },
+                lookup: binLookup,
+                bucket_parameters: result.bucketParameters
+            };
+            this.bucketParameters.push({
+                ...values,
+                index: LEGACY_BUCKET_PARAMETER_INDEX_ID
+            });
+            this.currentSize += 200;
+        }
+        for (let lookup of remaining_lookups.values()) {
+            const op_id = data.op_seq.next();
+            this.debugLastOpId = op_id;
+            const values = {
+                _id: op_id,
+                key: {
+                    g: this.group_id,
+                    t: mongoTableId(sourceTable.id),
+                    k: sourceKey
+                },
+                lookup,
+                bucket_parameters: []
+            };
+            this.bucketParameters.push({
+                ...values,
+                index: LEGACY_BUCKET_PARAMETER_INDEX_ID
+            });
+            this.currentSize += 200;
+        }
+    }
+    hardDeleteCurrentData(sourceTableId, replicaId) {
+        this.currentData.push({
+            deleteOne: {
+                filter: { _id: this.currentDataId(sourceTableId, replicaId) }
+            }
+        });
+        this.currentSize += 50;
+    }
+    softDeleteCurrentData(sourceTableId, replicaId, _checkpointGreaterThan) {
+        this.hardDeleteCurrentData(sourceTableId, replicaId);
+    }
+    upsertCurrentData(values) {
+        const buckets = values.buckets.map((bucket) => {
+            if (bucket.definitionId != null) {
+                throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled');
+            }
+            return {
+                bucket: bucket.bucket,
+                table: bucket.table,
+                id: bucket.id
+            };
+        });
+        const lookups = values.lookups.map((lookup) => {
+            if (lookup.indexId != null) {
+                throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled');
+            }
+            return lookup.lookup;
+        });
+        this.currentData.push({
+            updateOne: {
+                filter: { _id: this.currentDataId(values.sourceTableId, values.replicaId) },
+                update: {
+                    $set: {
+                        data: values.data ?? EMPTY_DATA,
+                        buckets,
+                        lookups
+                    }
+                },
+                upsert: true
+            }
+        });
+        this.currentSize += (values.data?.length() ?? 0) + 100;
+    }
+    get currentDataCount() {
+        return this.currentData.length;
+    }
+    async flushBucketData(session) {
+        await this.db.bucketDataV1.bulkWrite(this.bucketData.map((document) => ({
+            insertOne: {
+                document: serializeBucketDataV1(document)
+            }
+        })), {
+            session,
+            ordered: false
+        });
+    }
+    async flushBucketParameters(session) {
+        await this.db.parameterIndexV1.bulkWrite(this.bucketParameters.map((document) => ({
+            insertOne: {
+                document: taggedBucketParameterDocumentToV1(document)
+            }
+        })), {
+            session,
+            ordered: false
+        });
+    }
+    async flushCurrentData(session) {
+        if (this.currentData.length == 0) {
+            return;
+        }
+        await this.db.sourceRecordsV1.bulkWrite(this.currentData, {
+            session,
+            ordered: true
+        });
+    }
+    async flushBucketStates(session) {
+        await this.db.bucketStateV1.bulkWrite(this.getBucketStateUpdates(), {
+            session,
+            ordered: false
+        });
+    }
+    resetCurrentData() {
+        this.currentData = [];
+    }
+    getBucketStateUpdates() {
+        return Array.from(this.bucketStates.values()).map((state) => {
+            return {
+                updateOne: {
+                    filter: {
+                        _id: {
+                            g: this.group_id,
+                            b: state.bucket
+                        }
+                    },
+                    update: {
+                        $set: {
+                            last_op: state.lastOp
+                        },
+                        $inc: {
+                            'estimate_since_compact.count': state.incrementCount,
+                            'estimate_since_compact.bytes': state.incrementBytes
+                        }
+                    },
+                    upsert: true
+                }
+            };
+        });
+    }
+    currentDataId(sourceTableId, replicaId) {
+        return {
+            g: this.group_id,
+            t: sourceTableId,
+            k: replicaId
+        };
+    }
+}
+//# sourceMappingURL=PersistedBatchV1.js.map

package/dist/storage/implementation/v1/PersistedBatchV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PersistedBatchV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/PersistedBatchV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAIlD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAEL,cAAc,EAGf,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,gCAAgC,EAAE,gCAAgC,EAAa,MAAM,cAAc,CAAC;AAE7G,OAAO,EAIL,qBAAqB,EACrB,iCAAiC,EAClC,MAAM,aAAa,CAAC;AAErB,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAGlD,WAAW,GAAuD,EAAE,CAAC;IAE3D,iBAAiB,CAAC,aAAwC;QAClE,0DAA0D;QAC1D,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IAES,qBAAqB,CAAC,aAA+B;QAC7D,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IAED,iBAAiB,CAAC,IAA8B;QAC9C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAEzD,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,IAAI,yBAAyB,CAAC,+DAA+D,CAAC,CAAC;YACvG,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACzE,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAEvD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,MAAM,MAAM,GAA4B;gBACtC,GAAG,EAAE,KAAK;gBACV,GAAG,EAAE;oBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC/B,CAAC,EAAE,SAAS;iBACb;gBACD,MAAM,EAAE,SAAS;gBACjB,iBAAiB,EAAE,MAAM,CAAC,gBAAgB;aAC3C,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,GAAG,MAAM;gBACT,KAAK,EAAE,gCAAgC;aACxC,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,MAAM,MAAM,GAA4B;gBACtC,GAAG,EAAE,KAAK;gBACV,GAAG,EAAE;oBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC/B,CAAC,EAAE,SAAS;iBACb;gBACD,MAAM;gBACN,iBAAiB,EAAE,EAAE;aACtB,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,GAAG,MAAM;gBACT,KAAK,EAAE,gCAAgC;aACxC,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,qBAAqB,CAAC,aAA4B,EAAE,SAA4B;QAC9E,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,SAAS,CAAC,EAAE;aAC9D;SACF,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,qBAAqB,CAAC,aAA4B,EAAE,SAA4B,EAAE,sBAA8B;QAC9G,IAAI,CAAC,qBAAqB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,iBAAiB,CAAC,MAAgC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5C,IAAI,MAAM,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;gBAChC,MAAM,IAAI,yBAAyB,CAAC,+DAA+D,CAAC,CAAC;YACvG,CAAC;YACD,OAAO;gBACL,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;aACd,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5C,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,IAAI,yBAAyB,CAAC,+DAA+D,CAAC,CAAC;YACvG,CAAC;YACD,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE;gBAC3E,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,UAAU;wBAC/B,OAAO;wBACP,OAAO;qBACR;iBACF;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;IACjC,CAAC;IAES,KAAK,CAAC,eAAe,CAAC,OAA4B;QAC1D,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACjC,SAAS,EAAE;gBACT,QAAQ,EAAE,qBAAqB,CAAC,QAAQ,CAAC;aAC1C;SACF,CAAC,CAAC,EACH;YACE,OAAO;YACP,OAAO,EAAE,KAAK;SACf,CACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,qBAAqB,CAAC,OAA4B;QAChE,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,SAAS,CACtC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACvC,SAAS,EAAE;gBACT,QAAQ,EAAE,iCAAiC,CAAC,QAAQ,CAAC;aACtD;SACF,CAAC,CAAC,EACH;YACE,OAAO;YACP,OAAO,EAAE,KAAK;SACf,CACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,OAA4B;QAC3D,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE;YACxD,OAAO;YACP,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,OAA4B;QAC5D,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE;YAClE,OAAO;YACP,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;IACL,CAAC;IAES,gBAAgB;QACxB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;IACxB,CAAC;IAEO,qBAAqB;QAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAwB,EAAE,EAAE;YAC7E,OAAO;gBACL,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,KAAK,CAAC,MAAM;yBAChB;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,OAAO,EAAE,KAAK,CAAC,MAAM;yBACtB;wBACD,IAAI,EAAE;4BACJ,8BAA8B,EAAE,KAAK,CAAC,cAAc;4BACpD,8BAA8B,EAAE,KAAK,CAAC,cAAc;yBACrD;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aAC2D,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,aAA4B,EAAE,SAA4B;QAC9E,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,aAAa;YAChB,CAAC,EAAE,SAAS;SACb,CAAC;IACJ,CAAC;CACF"}