@powersync/service-core 0.0.0-dev-20241119082750 → 0.0.0-dev-20241219091224

package/dist/storage/mongo/MongoCompactor.d.ts

@@ -1,40 +0,0 @@
-import { PowerSyncMongo } from './db.js';
-import { CompactOptions } from '../BucketStorage.js';
-/**
- * Additional options, primarily for testing.
- */
-export interface MongoCompactOptions extends CompactOptions {
-    /** Minimum of 2 */
-    clearBatchLimit?: number;
-    /** Minimum of 1 */
-    moveBatchLimit?: number;
-    /** Minimum of 1 */
-    moveBatchQueryLimit?: number;
-}
-export declare class MongoCompactor {
-    private db;
-    private group_id;
-    private updates;
-    private idLimitBytes;
-    private moveBatchLimit;
-    private moveBatchQueryLimit;
-    private clearBatchLimit;
-    private maxOpId;
-    private buckets;
-    constructor(db: PowerSyncMongo, group_id: number, options?: MongoCompactOptions);
-    /**
-     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
-     *
-     * See /docs/compacting-operations.md for details.
-     */
-    compact(): Promise<void>;
-    compactInternal(bucket: string | undefined): Promise<void>;
-    private flush;
-    /**
-     * Perform a CLEAR compact for a bucket.
-     *
-     * @param bucket bucket name
-     * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
-     */
-    private clearBucket;
-}

package/dist/storage/mongo/MongoCompactor.js

@@ -1,309 +0,0 @@
-import { logger } from '@powersync/lib-services-framework';
-import { MaxKey, MinKey } from 'mongodb';
-import { addChecksums } from '../../util/utils.js';
-import { cacheKey } from './OperationBatch.js';
-const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
-const DEFAULT_MOVE_BATCH_LIMIT = 2000;
-const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10000;
-/** This default is primarily for tests. */
-const DEFAULT_MEMORY_LIMIT_MB = 64;
-export class MongoCompactor {
-    constructor(db, group_id, options) {
-        this.db = db;
-        this.group_id = group_id;
-        this.updates = [];
-        this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
-        this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
-        this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
-        this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
-        this.maxOpId = options?.maxOpId;
-        this.buckets = options?.compactBuckets;
-    }
-    /**
-     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
-     *
-     * See /docs/compacting-operations.md for details.
-     */
-    async compact() {
-        if (this.buckets) {
-            for (let bucket of this.buckets) {
-                // We can make this more efficient later on by iterating
-                // through the buckets in a single query.
-                // That makes batching more tricky, so we leave for later.
-                await this.compactInternal(bucket);
-            }
-        }
-        else {
-            await this.compactInternal(undefined);
-        }
-    }
-    async compactInternal(bucket) {
-        const idLimitBytes = this.idLimitBytes;
-        let currentState = null;
-        let bucketLower;
-        let bucketUpper;
-        if (bucket == null) {
-            bucketLower = new MinKey();
-            bucketUpper = new MaxKey();
-        }
-        else if (bucket.includes('[')) {
-            // Exact bucket name
-            bucketLower = bucket;
-            bucketUpper = bucket;
-        }
-        else {
-            // Bucket definition name
-            bucketLower = `${bucket}[`;
-            bucketUpper = `${bucket}[\uFFFF`;
-        }
-        // Constant lower bound
-        const lowerBound = {
-            g: this.group_id,
-            b: bucketLower,
-            o: new MinKey()
-        };
-        // Upper bound is adjusted for each batch
-        let upperBound = {
-            g: this.group_id,
-            b: bucketUpper,
-            o: new MaxKey()
-        };
-        while (true) {
-            // Query one batch at a time, to avoid cursor timeouts
-            const batch = await this.db.bucket_data
-                .find({
-                _id: {
-                    $gte: lowerBound,
-                    $lt: upperBound
-                }
-            }, {
-                projection: {
-                    _id: 1,
-                    op: 1,
-                    table: 1,
-                    row_id: 1,
-                    source_table: 1,
-                    source_key: 1
-                },
-                limit: this.moveBatchQueryLimit,
-                sort: { _id: -1 },
-                singleBatch: true
-            })
-                .toArray();
-            if (batch.length == 0) {
-                // We've reached the end
-                break;
-            }
-            // Set upperBound for the next batch
-            upperBound = batch[batch.length - 1]._id;
-            for (let doc of batch) {
-                if (currentState == null || doc._id.b != currentState.bucket) {
-                    if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
-                        // Important to flush before clearBucket()
-                        await this.flush();
-                        logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
-                        const bucket = currentState.bucket;
-                        const clearOp = currentState.lastNotPut;
-                        // Free memory before clearing bucket
-                        currentState = null;
-                        await this.clearBucket(bucket, clearOp);
-                    }
-                    currentState = {
-                        bucket: doc._id.b,
-                        seen: new Map(),
-                        trackingSize: 0,
-                        lastNotPut: null,
-                        opsSincePut: 0
-                    };
-                }
-                if (this.maxOpId != null && doc._id.o > this.maxOpId) {
-                    continue;
-                }
-                let isPersistentPut = doc.op == 'PUT';
-                if (doc.op == 'REMOVE' || doc.op == 'PUT') {
-                    const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table, doc.source_key)}`;
-                    const targetOp = currentState.seen.get(key);
-                    if (targetOp) {
-                        // Will convert to MOVE, so don't count as PUT
-                        isPersistentPut = false;
-                        this.updates.push({
-                            updateOne: {
-                                filter: {
-                                    _id: doc._id
-                                },
-                                update: {
-                                    $set: {
-                                        op: 'MOVE',
-                                        target_op: targetOp
-                                    },
-                                    $unset: {
-                                        source_table: 1,
-                                        source_key: 1,
-                                        table: 1,
-                                        row_id: 1,
-                                        data: 1
-                                    }
-                                }
-                            }
-                        });
-                    }
-                    else {
-                        if (currentState.trackingSize >= idLimitBytes) {
-                            // Reached memory limit.
-                            // Keep the highest seen values in this case.
-                        }
-                        else {
-                            // flatstr reduces the memory usage by flattening the string
-                            currentState.seen.set(flatstr(key), doc._id.o);
-                            // length + 16 for the string
-                            // 24 for the bigint
-                            // 50 for map overhead
-                            // 50 for additional overhead
-                            currentState.trackingSize += key.length + 140;
-                        }
-                    }
-                }
-                if (isPersistentPut) {
-                    currentState.lastNotPut = null;
-                    currentState.opsSincePut = 0;
-                }
-                else if (doc.op != 'CLEAR') {
-                    if (currentState.lastNotPut == null) {
-                        currentState.lastNotPut = doc._id.o;
-                    }
-                    currentState.opsSincePut += 1;
-                }
-                if (this.updates.length >= this.moveBatchLimit) {
-                    await this.flush();
-                }
-            }
-        }
-        await this.flush();
-        currentState?.seen.clear();
-        if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) {
-            logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
-            const bucket = currentState.bucket;
-            const clearOp = currentState.lastNotPut;
-            // Free memory before clearing bucket
-            currentState = null;
-            await this.clearBucket(bucket, clearOp);
-        }
-    }
-    async flush() {
-        if (this.updates.length > 0) {
-            logger.info(`Compacting ${this.updates.length} ops`);
-            await this.db.bucket_data.bulkWrite(this.updates, {
-                // Order is not important.
-                // Since checksums are not affected, these operations can happen in any order,
-                // and it's fine if the operations are partially applied.
-                // Each individual operation is atomic.
-                ordered: false
-            });
-            this.updates = [];
-        }
-    }
-    /**
-     * Perform a CLEAR compact for a bucket.
-     *
-     * @param bucket bucket name
-     * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
-     */
-    async clearBucket(bucket, op) {
-        const opFilter = {
-            _id: {
-                $gte: {
-                    g: this.group_id,
-                    b: bucket,
-                    o: new MinKey()
-                },
-                $lte: {
-                    g: this.group_id,
-                    b: bucket,
-                    o: op
-                }
-            }
-        };
-        const session = this.db.client.startSession();
-        try {
-            let done = false;
-            while (!done) {
-                // Do the CLEAR operation in batches, with each batch a separate transaction.
-                // The state after each batch is fully consistent.
-                // We need a transaction per batch to make sure checksums stay consistent.
-                await session.withTransaction(async () => {
-                    const query = this.db.bucket_data.find(opFilter, {
-                        session,
-                        sort: { _id: 1 },
-                        projection: {
-                            _id: 1,
-                            op: 1,
-                            checksum: 1,
-                            target_op: 1
-                        },
-                        limit: this.clearBatchLimit
-                    });
-                    let checksum = 0;
-                    let lastOpId = null;
-                    let targetOp = null;
-                    let gotAnOp = false;
-                    for await (let op of query.stream()) {
-                        if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
-                            checksum = addChecksums(checksum, op.checksum);
-                            lastOpId = op._id;
-                            if (op.op != 'CLEAR') {
-                                gotAnOp = true;
-                            }
-                            if (op.target_op != null) {
-                                if (targetOp == null || op.target_op > targetOp) {
-                                    targetOp = op.target_op;
-                                }
-                            }
-                        }
-                        else {
-                            throw new Error(`Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}`);
-                        }
-                    }
-                    if (!gotAnOp) {
-                        done = true;
-                        return;
-                    }
-                    logger.info(`Flushing CLEAR at ${lastOpId?.o}`);
-                    await this.db.bucket_data.deleteMany({
-                        _id: {
-                            $gte: {
-                                g: this.group_id,
-                                b: bucket,
-                                o: new MinKey()
-                            },
-                            $lte: lastOpId
-                        }
-                    }, { session });
-                    await this.db.bucket_data.insertOne({
-                        _id: lastOpId,
-                        op: 'CLEAR',
-                        checksum: checksum,
-                        data: null,
-                        target_op: targetOp
-                    }, { session });
-                }, {
-                    writeConcern: { w: 'majority' },
-                    readConcern: { level: 'snapshot' }
-                });
-            }
-        }
-        finally {
-            await session.endSession();
-        }
-    }
-}
-/**
- * Flattens string to reduce memory usage (around 320 bytes -> 120 bytes),
- * at the cost of some upfront CPU usage.
- *
- * From: https://github.com/davidmarkclements/flatstr/issues/8
- */
-function flatstr(s) {
-    s.match(/\n/g);
-    return s;
-}
-//# sourceMappingURL=MongoCompactor.js.map

package/dist/storage/mongo/MongoCompactor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAInD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAqC/C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAUzB,YACU,EAAkB,EAClB,QAAgB,EACxB,OAA6B;QAFrB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAXlB,YAAO,GAAgD,EAAE,CAAC;QAchE,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAA4B,CAAC;QACjC,IAAI,WAA4B,CAAC;QAEjC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,WAAW,GAAG,IAAI,MAAM,EAAE,CAAC;YAC3B,WAAW,GAAG,IAAI,MAAM,EAAE,CAAC;QAC7B,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,OAAO,IAAI,EAAE,CAAC;YACZ,sDAAsD;YACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACpC,IAAI,CACH;gBACE,GAAG,EAAE;oBACH,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB;aACF,EACD;gBACE,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,YAAY,EAAE,CAAC;oBACf,UAAU,EAAE,CAAC;iBACd;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB;gBAC/B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,WAAW,EAAE,IAAI;aAClB,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC7D,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;wBAC7F,0CAA0C;wBAC1C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;wBAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;wBACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;wBACxC,qCAAqC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC1C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;qBACf,CAAC;gBACJ,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BAC/C,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;YACxC,qCAAqC;YACrC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAU;QAClD,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,MAAM,EAAS;iBACvB;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,EAAE;iBACN;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;4BAC/C,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1F,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,MAAM,EAAS;6BACvB;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,SAAS,OAAO,CAAC,CAAS;IACxB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/storage/mongo/MongoIdSequence.d.ts

@@ -1,12 +0,0 @@
-/**
- * Manages op_id or similar sequence in memory.
- *
- * This is typically used within a transaction, with the last value persisted
- * at the end of the transaction.
- */
-export declare class MongoIdSequence {
-    private _last;
-    constructor(last: bigint);
-    next(): bigint;
-    last(): bigint;
-}

package/dist/storage/mongo/MongoIdSequence.js

@@ -1,21 +0,0 @@
-/**
- * Manages op_id or similar sequence in memory.
- *
- * This is typically used within a transaction, with the last value persisted
- * at the end of the transaction.
- */
-export class MongoIdSequence {
-    constructor(last) {
-        if (typeof last != 'bigint') {
-            throw new Error(`BigInt required, got ${last} ${typeof last}`);
-        }
-        this._last = last;
-    }
-    next() {
-        return ++this._last;
-    }
-    last() {
-        return this._last;
-    }
-}
-//# sourceMappingURL=MongoIdSequence.js.map

package/dist/storage/mongo/MongoIdSequence.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"MongoIdSequence.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoIdSequence.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IAG1B,YAAY,IAAY;QACtB,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,IAAI,OAAO,IAAI,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAI;QACF,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF"}

package/dist/storage/mongo/MongoPersistedSyncRules.d.ts

@@ -1,9 +0,0 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { PersistedSyncRules } from '../BucketStorage.js';
-export declare class MongoPersistedSyncRules implements PersistedSyncRules {
-    readonly id: number;
-    readonly sync_rules: SqlSyncRules;
-    readonly checkpoint_lsn: string | null;
-    readonly slot_name: string;
-    constructor(id: number, sync_rules: SqlSyncRules, checkpoint_lsn: string | null, slot_name: string | null);
-}

package/dist/storage/mongo/MongoPersistedSyncRules.js

@@ -1,9 +0,0 @@
-export class MongoPersistedSyncRules {
-    constructor(id, sync_rules, checkpoint_lsn, slot_name) {
-        this.id = id;
-        this.sync_rules = sync_rules;
-        this.checkpoint_lsn = checkpoint_lsn;
-        this.slot_name = slot_name ?? `powersync_${id}`;
-    }
-}
-//# sourceMappingURL=MongoPersistedSyncRules.js.map

package/dist/storage/mongo/MongoPersistedSyncRules.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"MongoPersistedSyncRules.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoPersistedSyncRules.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,uBAAuB;IAGlC,YACkB,EAAU,EACV,UAAwB,EACxB,cAA6B,EAC7C,SAAwB;QAHR,OAAE,GAAF,EAAE,CAAQ;QACV,eAAU,GAAV,UAAU,CAAc;QACxB,mBAAc,GAAd,cAAc,CAAe;QAG7C,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,aAAa,EAAE,EAAE,CAAC;IAClD,CAAC;CACF"}

package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts

@@ -1,20 +0,0 @@
-import * as mongo from 'mongodb';
-import { ParseSyncRulesOptions, PersistedSyncRulesContent } from '../BucketStorage.js';
-import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
-import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
-import { PowerSyncMongo } from './db.js';
-import { SyncRuleDocument } from './models.js';
-export declare class MongoPersistedSyncRulesContent implements PersistedSyncRulesContent {
-    private db;
-    readonly slot_name: string;
-    readonly id: number;
-    readonly sync_rules_content: string;
-    readonly last_checkpoint_lsn: string | null;
-    readonly last_fatal_error: string | null;
-    readonly last_keepalive_ts: Date | null;
-    readonly last_checkpoint_ts: Date | null;
-    current_lock: MongoSyncRulesLock | null;
-    constructor(db: PowerSyncMongo, doc: mongo.WithId<SyncRuleDocument>);
-    parsed(options: ParseSyncRulesOptions): MongoPersistedSyncRules;
-    lock(): Promise<MongoSyncRulesLock>;
-}

package/dist/storage/mongo/MongoPersistedSyncRulesContent.js

@@ -1,26 +0,0 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
-import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
-export class MongoPersistedSyncRulesContent {
-    constructor(db, doc) {
-        this.db = db;
-        this.current_lock = null;
-        this.id = doc._id;
-        this.sync_rules_content = doc.content;
-        this.last_checkpoint_lsn = doc.last_checkpoint_lsn;
-        // Handle legacy values
-        this.slot_name = doc.slot_name ?? `powersync_${this.id}`;
-        this.last_fatal_error = doc.last_fatal_error;
-        this.last_checkpoint_ts = doc.last_checkpoint_ts;
-        this.last_keepalive_ts = doc.last_keepalive_ts;
-    }
-    parsed(options) {
-        return new MongoPersistedSyncRules(this.id, SqlSyncRules.fromYaml(this.sync_rules_content, options), this.last_checkpoint_lsn, this.slot_name);
-    }
-    async lock() {
-        const lock = await MongoSyncRulesLock.createLock(this.db, this);
-        this.current_lock = lock;
-        return lock;
-    }
-}
-//# sourceMappingURL=MongoPersistedSyncRulesContent.js.map

package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"MongoPersistedSyncRulesContent.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoPersistedSyncRulesContent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAI7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAI7D,MAAM,OAAO,8BAA8B;IAYzC,YACU,EAAkB,EAC1B,GAAmC;QAD3B,OAAE,GAAF,EAAE,CAAgB;QAHrB,iBAAY,GAA8B,IAAI,CAAC;QAMpD,IAAI,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC;QAClB,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAC,OAAO,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC,mBAAmB,CAAC;QACnD,uBAAuB;QACvB,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC,gBAAgB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAC,kBAAkB,CAAC;QACjD,IAAI,CAAC,iBAAiB,GAAG,GAAG,CAAC,iBAAiB,CAAC;IACjD,CAAC;IAED,MAAM,CAAC,OAA8B;QACnC,OAAO,IAAI,uBAAuB,CAChC,IAAI,CAAC,EAAE,EACP,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,OAAO,CAAC,EACvD,IAAI,CAAC,mBAAmB,EACxB,IAAI,CAAC,SAAS,CACf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/storage/mongo/MongoStorageProvider.d.ts

@@ -1,5 +0,0 @@
-import { ActiveStorage, BucketStorageProvider, GetStorageOptions } from '../StorageProvider.js';
-export declare class MongoStorageProvider implements BucketStorageProvider {
-    get type(): string;
-    getStorage(options: GetStorageOptions): Promise<ActiveStorage>;
-}

package/dist/storage/mongo/MongoStorageProvider.js

@@ -1,26 +0,0 @@
-import { logger } from '@powersync/lib-services-framework';
-import * as db from '../../db/db-index.js';
-import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { PowerSyncMongo } from './db.js';
-export class MongoStorageProvider {
-    get type() {
-        return 'mongodb';
-    }
-    async getStorage(options) {
-        const { resolvedConfig } = options;
-        const client = db.mongo.createMongoClient(resolvedConfig.storage);
-        const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
-        return {
-            storage: new MongoBucketStorage(database, {
-                // TODO currently need the entire resolved config due to this
-                slot_name_prefix: resolvedConfig.slot_name_prefix
-            }),
-            shutDown: () => client.close(),
-            tearDown: () => {
-                logger.info(`Tearing down storage: ${database.db.namespace}...`);
-                return database.db.dropDatabase();
-            }
-        };
-    }
-}
-//# sourceMappingURL=MongoStorageProvider.js.map

package/dist/storage/mongo/MongoStorageProvider.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"MongoStorageProvider.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoStorageProvider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,MAAM,OAAO,oBAAoB;IAC/B,IAAI,IAAI;QACN,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAA0B;QACzC,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEnC,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAElE,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE3F,OAAO;YACL,OAAO,EAAE,IAAI,kBAAkB,CAAC,QAAQ,EAAE;gBACxC,6DAA6D;gBAC7D,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;aAClD,CAAC;YACF,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE;YAC9B,QAAQ,EAAE,GAAG,EAAE;gBACb,MAAM,CAAC,IAAI,CAAC,yBAAyB,QAAQ,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC;gBACjE,OAAO,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;YACpC,CAAC;SACsB,CAAC;IAC5B,CAAC;CACF"}

package/dist/storage/mongo/MongoSyncBucketStorage.d.ts

@@ -1,38 +0,0 @@
-import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules';
-import { DisposableObserver } from '@powersync/lib-services-framework';
-import * as util from '../../util/util-index.js';
-import { BucketDataBatchOptions, BucketStorageBatch, CompactOptions, FlushedResult, ParseSyncRulesOptions, PersistedSyncRulesContent, ReplicationCheckpoint, ResolveTableOptions, ResolveTableResult, StartBatchOptions, SyncBucketDataBatch, SyncRulesBucketStorage, SyncRulesBucketStorageListener, SyncRuleStatus, TerminateOptions } from '../BucketStorage.js';
-import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { BatchedCustomWriteCheckpointOptions, ManagedWriteCheckpointOptions, SyncStorageLastWriteCheckpointFilters, WriteCheckpointMode } from '../WriteCheckpointAPI.js';
-export declare class MongoSyncBucketStorage extends DisposableObserver<SyncRulesBucketStorageListener> implements SyncRulesBucketStorage {
-    readonly factory: MongoBucketStorage;
-    readonly group_id: number;
-    private readonly sync_rules;
-    readonly slot_name: string;
-    private readonly db;
-    private checksumCache;
-    private parsedSyncRulesCache;
-    private writeCheckpointAPI;
-    constructor(factory: MongoBucketStorage, group_id: number, sync_rules: PersistedSyncRulesContent, slot_name: string, writeCheckpointMode?: WriteCheckpointMode);
-    get writeCheckpointMode(): WriteCheckpointMode;
-    setWriteCheckpointMode(mode: WriteCheckpointMode): void;
-    batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
-    createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
-    createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
-    lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
-    getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules;
-    getCheckpoint(): Promise<ReplicationCheckpoint>;
-    startBatch(options: StartBatchOptions, callback: (batch: BucketStorageBatch) => Promise<void>): Promise<FlushedResult | null>;
-    resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
-    getParameterSets(checkpoint: util.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]>;
-    getBucketDataBatch(checkpoint: util.OpId, dataBuckets: Map<string, string>, options?: BucketDataBatchOptions): AsyncIterable<SyncBucketDataBatch>;
-    getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
-    private getChecksumsInternal;
-    terminate(options?: TerminateOptions): Promise<void>;
-    getStatus(): Promise<SyncRuleStatus>;
-    clear(): Promise<void>;
-    private clearIteration;
-    autoActivate(): Promise<void>;
-    reportError(e: any): Promise<void>;
-    compact(options?: CompactOptions): Promise<void>;
-}