@powersync/service-module-mongodb-storage 0.23.1 → 0.24.0

package/dist/storage/implementation/common/PersistedBatch.js

@@ -1,292 +1,2 @@
-import { logger as defaultLogger } from '@powersync/lib-services-framework';
-import { utils } from '@powersync/service-core';
-import { JSONBig } from '@powersync/service-jsonbig';
-import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
-import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
-/**
- * Maximum size of operations we write in a single transaction.
- *
- * It's tricky to find the exact limit, but from experience, over 100MB
- * can cause an error:
- * > transaction is too large and will not fit in the storage engine cache
- *
- * Additionally, unbounded size here can balloon our memory usage in some edge
- * cases.
- *
- * When we reach this threshold, we commit the transaction and start a new one.
- */
-const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
-/**
- * Limit number of documents to write in a single transaction.
- *
- * This has an effect on error message size in some cases.
- */
-const MAX_TRANSACTION_DOC_COUNT = 2_000;
-export const DEFAULT_INLINE_THRESHOLD_BYTES = 16 * 1024;
-/**
- * Keeps track of bulkwrite operations within a transaction.
- *
- * There may be multiple of these batches per transaction, but it may not span
- * multiple transactions.
- */
-export class PersistedBatch {
-    db;
-    group_id;
-    mapping;
-    logger;
-    bucketData = [];
-    bucketParameters = [];
-    bucketStates = new Map();
-    objectStorage;
-    inlineThresholdBytes = DEFAULT_INLINE_THRESHOLD_BYTES;
-    signal;
-    objectStorageUsageWriterId;
-    /**
-     * For debug logging only.
-     */
-    debugLastOpId = null;
-    /**
-     * Very rough estimate of transaction size.
-     */
-    currentSize = 0;
-    constructor(db, group_id, mapping, writtenSize, options) {
-        this.db = db;
-        this.group_id = group_id;
-        this.mapping = mapping;
-        this.currentSize = writtenSize;
-        this.logger = options?.logger ?? defaultLogger;
-        this.objectStorage = options?.objectStorage;
-        this.signal = options?.signal;
-        this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
-        if (options?.inlineThresholdBytes != null) {
-            this.inlineThresholdBytes = options.inlineThresholdBytes;
-        }
-    }
-    saveBucketData(options) {
-        const remaining_buckets = new Map();
-        for (let bucket of options.before_buckets) {
-            const mapped = {
-                bucket: bucket.bucket,
-                definitionId: this.checkDefinitionId(bucket.definitionId),
-                id: bucket.id,
-                table: bucket.table
-            };
-            if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId)) {
-                // The bucket definition is not active anymore.
-                // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
-                continue;
-            }
-            remaining_buckets.set(currentBucketKey(mapped), mapped);
-        }
-        const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
-        const dchecksum = BigInt(utils.hashDelete(subkey));
-        for (const evaluated of options.evaluated) {
-            const definitionId = this.getBucketDefinitionId(evaluated.source);
-            const key = currentBucketKey({
-                definitionId: definitionId,
-                bucket: evaluated.bucket,
-                table: evaluated.table,
-                id: evaluated.id
-            });
-            const recordData = JSONBig.stringify(evaluated.data);
-            const checksum = utils.hashData(evaluated.table, evaluated.id, recordData);
-            if (recordData.length > MAX_ROW_SIZE) {
-                this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`);
-                continue;
-            }
-            remaining_buckets.delete(key);
-            const byteEstimate = recordData.length + 200;
-            this.currentSize += byteEstimate;
-            const op_id = options.op_seq.next();
-            this.debugLastOpId = op_id;
-            this.addBucketDataPut({
-                bucketKey: {
-                    bucket: evaluated.bucket,
-                    definitionId: definitionId,
-                    replicationStreamId: this.group_id
-                },
-                op_id,
-                bucket: evaluated.bucket,
-                sourceTableId: options.table.id,
-                sourceKey: options.sourceKey,
-                subkey,
-                table: evaluated.table,
-                rowId: evaluated.id,
-                checksum: BigInt(checksum),
-                data: recordData
-            });
-            this.incrementBucket(definitionId, evaluated.bucket, op_id, byteEstimate);
-        }
-        for (let bucket of remaining_buckets.values()) {
-            const definitionId = bucket.definitionId;
-            const op_id = options.op_seq.next();
-            this.debugLastOpId = op_id;
-            this.addBucketDataRemove({
-                bucketKey: {
-                    replicationStreamId: this.group_id,
-                    definitionId,
-                    bucket: bucket.bucket
-                },
-                op_id,
-                sourceTableId: options.table.id,
-                sourceKey: options.sourceKey,
-                subkey,
-                table: bucket.table,
-                rowId: bucket.id,
-                checksum: dchecksum
-            });
-            this.currentSize += 200;
-            this.incrementBucket(definitionId, bucket.bucket, op_id, 200);
-        }
-    }
-    /**
-     * New bucket operations persist protocol subkeys during replication. Older
-     * V1 documents may not have one, so sync reads retain a fallback.
-     */
-    persistedSubkey(table, key) {
-        return replicaIdToSubkey(table, key);
-    }
-    get bucketDataCount() {
-        return this.bucketData.length;
-    }
-    incrementBucket(definitionId, bucket, op_id, bytes) {
-        const key = `${definitionId ?? ''}:${bucket}`;
-        let existingState = this.bucketStates.get(key);
-        if (existingState) {
-            existingState.lastOp = op_id;
-            existingState.incrementCount += 1;
-            existingState.incrementBytes += bytes;
-        }
-        else {
-            this.bucketStates.set(key, {
-                definitionId,
-                bucket,
-                lastOp: op_id,
-                incrementCount: 1,
-                incrementBytes: bytes,
-                incrementChunks: 0
-            });
-        }
-    }
-    /**
-     * V3 persists operations in chunks. Keep this separate from incrementBucket:
-     * operation counts are known while evaluating rows, while chunk counts and
-     * exact persisted bytes are only known after the writer has chunked a flush.
-     */
-    incrementBucketPersistedChunk(definitionId, bucket, bytes) {
-        const key = `${definitionId ?? ''}:${bucket}`;
-        const existingState = this.bucketStates.get(key);
-        if (existingState != null) {
-            existingState.incrementChunks += 1;
-            existingState.incrementBytes += bytes;
-        }
-    }
-    /**
-     * V3's compactor calculates byte deltas from persisted chunk metadata.
-     * Replace the writer's per-operation estimate with those exact sizes before
-     * flushing the corresponding bucket state.
-     */
-    resetBucketPersistedBytes(definitionId, bucket) {
-        const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
-        if (state != null) {
-            state.incrementBytes = 0;
-        }
-    }
-    addBucketDataPut(options) {
-        this.bucketData.push({
-            bucketKey: options.bucketKey,
-            o: options.op_id,
-            op: 'PUT',
-            source_table: mongoTableId(options.sourceTableId),
-            source_key: options.sourceKey,
-            subkey: options.subkey,
-            table: options.table,
-            row_id: options.rowId,
-            checksum: options.checksum,
-            data: options.data
-        });
-    }
-    addBucketDataRemove(options) {
-        this.bucketData.push({
-            bucketKey: options.bucketKey,
-            o: options.op_id,
-            op: 'REMOVE',
-            source_table: mongoTableId(options.sourceTableId),
-            source_key: options.sourceKey,
-            subkey: options.subkey,
-            table: options.table,
-            row_id: options.rowId,
-            checksum: options.checksum,
-            data: null
-        });
-    }
-    shouldFlushTransaction() {
-        return (this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
-            this.bucketDataCount >= MAX_TRANSACTION_DOC_COUNT ||
-            this.currentDataCount >= MAX_TRANSACTION_DOC_COUNT ||
-            this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT);
-    }
-    async flush(session, options) {
-        const startAt = performance.now();
-        let flushedSomething = false;
-        const writes = this.db.createWriteBatch(session, { ordered: false });
-        if (this.bucketDataCount > 0) {
-            flushedSomething = true;
-            await this.queueBucketData(writes);
-        }
-        if (this.bucketParameters.length > 0) {
-            flushedSomething = true;
-            this.queueBucketParameters(writes);
-        }
-        if (this.currentDataCount > 0) {
-            flushedSomething = true;
-            this.queueCurrentData(writes);
-        }
-        if (this.bucketStates.size > 0) {
-            flushedSomething = true;
-            this.queueBucketStates(writes);
-        }
-        await writes.execute();
-        if (flushedSomething) {
-            const duration = Math.round(performance.now() - startAt);
-            if (options?.oldestUncommittedChange != null) {
-                const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
-                this.logger.info(`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${this.currentDataCount} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, {
-                    flushed: {
-                        duration: duration,
-                        size: this.currentSize,
-                        bucket_ops_count: this.bucketDataCount,
-                        parameter_indexes_count: this.bucketParameters.length,
-                        source_records_count: this.currentDataCount,
-                        replication_lag_seconds: replicationLag
-                    }
-                });
-            }
-            else {
-                this.logger.info(`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${this.currentDataCount} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, {
-                    flushed: {
-                        duration: duration,
-                        size: this.currentSize,
-                        bucket_ops_count: this.bucketDataCount,
-                        parameter_indexes_count: this.bucketParameters.length,
-                        source_records_count: this.currentDataCount
-                    }
-                });
-            }
-        }
-        const stats = {
-            bucketDataCount: this.bucketDataCount,
-            parameterDataCount: this.bucketParameters.length,
-            currentDataCount: this.currentDataCount,
-            flushedAny: flushedSomething
-        };
-        this.bucketData = [];
-        this.bucketParameters = [];
-        this.resetCurrentData();
-        this.bucketStates.clear();
-        this.currentSize = 0;
-        this.debugLastOpId = null;
-        return stats;
-    }
-}
+export {};
 //# sourceMappingURL=PersistedBatch.js.map

package/dist/storage/implementation/common/PersistedBatch.js.map

@@ -1 +1 @@
-{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAkD,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAS9E;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,IAAI,CAAC;AAqCxD;;;;;GAKG;AACH,MAAM,OAAgB,cAAc;IAsBb;IACA;IACA;IAvBrB,MAAM,CAAS;IACf,UAAU,GAAoB,EAAE,CAAC;IACjC,gBAAgB,GAAoC,EAAE,CAAC;IACvD,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEtC,aAAa,CAAiB;IAC9B,oBAAoB,GAAW,8BAA8B,CAAC;IAC9D,MAAM,CAAe;IACrB,0BAA0B,CAAU;IAEvD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAAgC,EACnD,WAAmB,EACnB,OAA+B;QAJZ,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAyB;QAInD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC9B,IAAI,CAAC,0BAA0B,GAAG,OAAO,EAAE,0BAA0B,CAAC;QACtE,IAAI,OAAO,EAAE,oBAAoB,IAAI,IAAI,EAAE,CAAC;YAC1C,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAA8B;QAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAmC,CAAC;QACrE,KAAK,IAAI,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,MAAM,GAA4B;gBACtC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;gBACzD,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,IAAI,OAAO,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,YAAa,CAAC,EAAE,CAAC;gBAC9G,+CAA+C;gBAC/C,yGAAyG;gBACzG,SAAS;YACX,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnD,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM,GAAG,GAAG,gBAAgB,CAAC;gBAC3B,YAAY,EAAE,YAAY;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,EAAE,EAAE,SAAS,CAAC,EAAE;aACjB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC;YAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,gBAAgB,CAAC;gBACpB,SAAS,EAAE;oBACT,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,YAAY,EAAE,YAAY;oBAC1B,mBAAmB,EAAE,IAAI,CAAC,QAAQ;iBACnC;gBACD,KAAK;gBACL,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,KAAK,EAAE,SAAS,CAAC,EAAE;gBACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC1B,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5E,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAa,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,mBAAmB,CAAC;gBACvB,SAAS,EAAE;oBACT,mBAAmB,EAAE,IAAI,CAAC,QAAQ;oBAClC,YAAY;oBACZ,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB;gBACD,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IA6BD;;;OAGG;IACO,eAAe,CAAC,KAA4B,EAAE,GAAsB;QAC5E,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,IAAc,eAAe;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAES,eAAe,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAmB,EAAE,KAAa;QAC5G,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;YAClC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzB,YAAY;gBACZ,MAAM;gBACN,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,KAAK;gBACrB,eAAe,EAAE,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,6BAA6B,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAa;QACrG,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,aAAa,CAAC,eAAe,IAAI,CAAC,CAAC;YACnC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,yBAAyB,CAAC,YAAgC,EAAE,MAAc;QAClF,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAES,gBAAgB,CAAC,OAW1B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,KAAK;YACT,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;IACL,CAAC;IAES,mBAAmB,CAAC,OAS7B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,QAAQ;YACZ,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,eAAe,IAAI,yBAAyB;YACjD,IAAI,CAAC,gBAAgB,IAAI,yBAAyB;YAClD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAA4B,EAAE,OAA0C;QAClF,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACrE,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,gBAAgB,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC9B,gBAAgB,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAgB,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QAEvB,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,EAAE,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAEnG,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,sBAAsB,cAAc,GAAG,EAC7I;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;wBAC3C,uBAAuB,EAAE,cAAc;qBACxC;iBACF,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,EAAE,EACxG;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;qBAC5C;iBACF,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAChD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAE1B,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
+{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":""}

package/dist/storage/implementation/common/PreparedPublication.d.ts

@@ -0,0 +1,43 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
+export interface PublicationStats {
+    bucketDataCount: number;
+    parameterDataCount: number;
+    currentDataCount: number;
+    flushedAny: boolean;
+}
+/**
+ * Sealed view of a v3/v4 batch. Preparation runs once, including on failure;
+ * publication creates fresh MongoDB writes on every attempt without clearing it.
+ *
+ * Scheduling metadata belongs to the caller: lease, selected config/mappings,
+ * input suffix, checkpoint intent and durable progress. Replay is valid only
+ * while that context and upload protection remain valid. A changed owner or
+ * config, or expired payload, must abandon this generation, never mutate it to
+ * fit a different context or overwrite its object paths. Publication also
+ * asserts that assigned IDs advance the freshly fenced persisted head.
+ *
+ * No application repair callbacks run here. The coordinator collects
+ * markRecordUnavailable records during evaluation and delivers them only after
+ * confirmed commit. Diagnostics run during evaluation.
+ * Before/after-flush hooks, source events and custom checkpoint consumption also
+ * belong to that coordinator, never to replayable writes. Only a confirmed
+ * commit may consume input, advance durable progress or notify readers.
+ */
+export interface PreparedPublication {
+    /**
+     * Explicitly start preparation after consuming assigned IDs. Sealing itself
+     * starts no I/O, so the coordinator can establish ownership before yielding.
+     * A rejected preparation is terminal for this generation; all started uploads
+     * must have settled before this promise rejects.
+     */
+    preparePayloads(): Promise<void>;
+    /**
+     * Requires completed readiness before starting the publication transaction.
+     * An optional batch supplies independent head/checkpoint writes to execute
+     * together with this group's data. This method owns execution of that batch;
+     * callers must not execute it again or include another group's dependent writes.
+     */
+    publish(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions, additionalWrites?: MongoWriteBatch): Promise<PublicationStats>;
+}

package/dist/storage/implementation/common/PreparedPublication.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=PreparedPublication.js.map

package/dist/storage/implementation/common/PreparedPublication.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PreparedPublication.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PreparedPublication.ts"],"names":[],"mappings":""}

package/dist/storage/implementation/common/SourceRecordStore.d.ts

@@ -1,5 +1,4 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { Logger } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { BucketDefinitionId, EvaluatedParameters, EvaluatedRow, ParameterIndexId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
@@ -31,5 +30,4 @@ export interface SourceRecordStore {
     loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise<Map<string, number>>;
     loadDocuments(session: mongo.ClientSession, entries: SourceRecordLookupEntry[], idsOnly: boolean): Promise<Map<string, LoadedSourceRecord>>;
     loadTruncateBatch(session: mongo.ClientSession, sourceTableId: bson.ObjectId, limit: number): Promise<LoadedSourceRecord[]>;
-    postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise<void>;
 }

package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts

@@ -2,9 +2,9 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { InternalOpId, SourceTable, storage } from '@powersync/service-core';
 import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
 import { MongoWriteBatch } from '../MongoWriteBatch.js';
-import { PersistedBatch } from '../common/PersistedBatch.js';
 import { SourceRecordStore } from '../common/SourceRecordStore.js';
 import { SyncRuleDocumentBase } from '../models.js';
+import { PersistedBatchV1 } from './PersistedBatchV1.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 export declare class MongoBucketBatchV1 extends MongoBucketBatch {
     readonly db: VersionedPowerSyncMongoV1;
@@ -19,28 +19,32 @@ export declare class MongoBucketBatchV1 extends MongoBucketBatch {
      */
     private persisted_op;
     constructor(options: MongoBucketBatchOptions);
+    private withReplicationTransaction;
+    private clearError;
+    protected flushInner(options?: storage.BatchBucketFlushOptions, checkpoint?: (stream: SyncRuleDocumentBase, lastOp: InternalOpId, writes?: MongoWriteBatch) => Promise<void>, projection?: mongo.Document): Promise<storage.FlushedResult | null>;
+    private replicateBatch;
+    protected truncateSingleInternal(sourceTable: storage.SourceTable): Promise<InternalOpId>;
     protected persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId;
     protected onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void;
-    protected recordPersistedOp(lastOp: InternalOpId): void;
-    protected createPersistedBatch(writtenSize: number): PersistedBatch;
+    private recordPersistedOp;
+    protected createPersistedBatch(writtenSize: number): PersistedBatchV1;
     protected get sourceRecordStore(): SourceRecordStore;
     protected cleanupDroppedSourceTables(_tables: SourceTable[]): Promise<void>;
-    protected batchCreateCustomWriteCheckpoints(session: mongo.ClientSession, opId: InternalOpId): Promise<void>;
-    resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
+    private batchCreateCustomWriteCheckpoints;
+    protected resolveTablesInternal(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
     /**
      * Build a SourceTable from its persisted v1 record.
      */
     private hydrateSourceTable;
-    getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
-    commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
-    keepalive(lsn: string): Promise<storage.CheckpointResult>;
-    setResumeLsn(lsn: string): Promise<void>;
-    markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
+    protected getSourceTableStatusInternal(table: storage.SourceTable): Promise<storage.SourceTable | null>;
+    protected commitInternal(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
+    protected setResumeLsnInternal(lsn: string): Promise<void>;
+    protected markAllSnapshotDoneInternal(no_checkpoint_before_lsn: string): Promise<void>;
     private snapshotDoneUpdate;
-    markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
+    protected markSnapshotDoneInternal(no_checkpoint_before_lsn: string, options?: {
         throwOnConflict?: boolean;
     }): Promise<void>;
-    markTableSnapshotRequired(_table: storage.SourceTable): Promise<void>;
-    markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
+    protected markTableSnapshotRequiredInternal(_table: storage.SourceTable): Promise<void>;
+    protected markTableSnapshotDoneInternal(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
     private autoActivate;
 }