@powersync/service-module-mongodb-storage 0.23.2 → 0.24.1

package/dist/storage/implementation/common/BatchOperationBuffer.d.ts

@@ -0,0 +1,79 @@
+import { Logger } from '@powersync/lib-services-framework';
+import { InternalOpId, storage } from '@powersync/service-core';
+import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
+import { TaggedBucketParameterDocument } from '../models.js';
+import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
+import { BucketStateUpdate, SaveBucketDataOptions } from './PersistedBatch.js';
+import { PublicationStats } from './PreparedPublication.js';
+/**
+ * Shared operation generation, size accounting and diagnostics. Each storage
+ * version owns this buffer and decides when to write, retain or clear its data.
+ * No transaction, sealing or payload-preparation state lives here.
+ */
+export declare class BatchOperationBuffer {
+    private readonly group_id;
+    private readonly definitions;
+    private readonly logger;
+    bucketData: BucketDataDoc[];
+    bucketParameters: TaggedBucketParameterDocument[];
+    bucketStates: Map<string, BucketStateUpdate>;
+    /**
+     * For debug logging only.
+     */
+    debugLastOpId: InternalOpId | null;
+    /**
+     * Very rough estimate of transaction size.
+     */
+    currentSize: number;
+    constructor(group_id: number, writtenSize: number, definitions: {
+        checkDefinitionId(id: BucketDefinitionId | null): BucketDefinitionId;
+        getBucketDefinitionId(source: BucketDataSource): BucketDefinitionId;
+    }, logger?: Logger);
+    saveBucketData(options: SaveBucketDataOptions): void;
+    /**
+     * New bucket operations persist protocol subkeys during replication. Older
+     * V1 documents may not have one, so sync reads retain a fallback.
+     */
+    persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string;
+    get bucketDataCount(): number;
+    incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number): void;
+    /**
+     * V3 persists operations in chunks. Keep this separate from incrementBucket:
+     * operation counts are known while evaluating rows, while chunk counts and
+     * exact persisted bytes are only known after the writer has chunked a flush.
+     */
+    incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number): void;
+    /**
+     * V3's compactor calculates byte deltas from persisted chunk metadata.
+     * Replace the writer's per-operation estimate with those exact sizes before
+     * flushing the corresponding bucket state.
+     */
+    resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string): void;
+    addBucketDataPut(options: {
+        op_id: InternalOpId;
+        bucketKey: BucketKey;
+        bucket: string;
+        sourceTableId: storage.SourceTable['id'];
+        sourceKey: storage.ReplicaId;
+        subkey: string;
+        table: string;
+        rowId: string;
+        checksum: bigint;
+        data: string;
+    }): void;
+    addBucketDataRemove(options: {
+        op_id: InternalOpId;
+        bucketKey: BucketKey;
+        sourceTableId: storage.SourceTable['id'];
+        sourceKey: storage.ReplicaId;
+        subkey: string;
+        table: string;
+        rowId: string;
+        checksum: bigint;
+    }): void;
+    shouldFlushTransaction(currentDataCount: number): boolean;
+    /** Adopt one fully evaluated row; unfinished rows never mutate this buffer. */
+    append(row: BatchOperationBuffer): void;
+    clear(): void;
+    logFlush(startAt: number, currentDataCount: number, options?: storage.BucketBatchCommitOptions): PublicationStats;
+}

package/dist/storage/implementation/common/BatchOperationBuffer.js

@@ -0,0 +1,286 @@
+import { logger as defaultLogger } from '@powersync/lib-services-framework';
+import { utils } from '@powersync/service-core';
+import { JSONBig } from '@powersync/service-jsonbig';
+import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
+import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
+/**
+ * Maximum size of operations we write in a single transaction.
+ *
+ * It's tricky to find the exact limit, but from experience, over 100MB
+ * can cause an error:
+ * > transaction is too large and will not fit in the storage engine cache
+ *
+ * Additionally, unbounded size here can balloon our memory usage in some edge
+ * cases.
+ *
+ * When we reach this threshold, we commit the transaction and start a new one.
+ */
+const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
+/**
+ * Limit number of documents to write in a single transaction.
+ *
+ * This has an effect on error message size in some cases.
+ */
+const MAX_TRANSACTION_DOC_COUNT = 2_000;
+/**
+ * Shared operation generation, size accounting and diagnostics. Each storage
+ * version owns this buffer and decides when to write, retain or clear its data.
+ * No transaction, sealing or payload-preparation state lives here.
+ */
+export class BatchOperationBuffer {
+    group_id;
+    definitions;
+    logger;
+    bucketData = [];
+    bucketParameters = [];
+    bucketStates = new Map();
+    /**
+     * For debug logging only.
+     */
+    debugLastOpId = null;
+    /**
+     * Very rough estimate of transaction size.
+     */
+    currentSize;
+    constructor(group_id, writtenSize, definitions, logger = defaultLogger) {
+        this.group_id = group_id;
+        this.definitions = definitions;
+        this.logger = logger;
+        this.currentSize = writtenSize;
+    }
+    saveBucketData(options) {
+        const remaining_buckets = new Map();
+        for (let bucket of options.before_buckets) {
+            const mapped = {
+                bucket: bucket.bucket,
+                definitionId: this.definitions.checkDefinitionId(bucket.definitionId),
+                id: bucket.id,
+                table: bucket.table
+            };
+            if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId)) {
+                // The bucket definition is not active anymore.
+                // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
+                continue;
+            }
+            remaining_buckets.set(currentBucketKey(mapped), mapped);
+        }
+        const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
+        const dchecksum = BigInt(utils.hashDelete(subkey));
+        for (const evaluated of options.evaluated) {
+            const definitionId = this.definitions.getBucketDefinitionId(evaluated.source);
+            const key = currentBucketKey({
+                definitionId: definitionId,
+                bucket: evaluated.bucket,
+                table: evaluated.table,
+                id: evaluated.id
+            });
+            const recordData = JSONBig.stringify(evaluated.data);
+            const checksum = utils.hashData(evaluated.table, evaluated.id, recordData);
+            if (recordData.length > MAX_ROW_SIZE) {
+                const message = `Row ${key} too large: ${recordData.length} bytes. Removing.`;
+                this.logger.error(message);
+                continue;
+            }
+            remaining_buckets.delete(key);
+            const byteEstimate = recordData.length + 200;
+            this.currentSize += byteEstimate;
+            const op_id = options.op_seq.next();
+            this.debugLastOpId = op_id;
+            this.addBucketDataPut({
+                bucketKey: {
+                    bucket: evaluated.bucket,
+                    definitionId: definitionId,
+                    replicationStreamId: this.group_id
+                },
+                op_id,
+                bucket: evaluated.bucket,
+                sourceTableId: options.table.id,
+                sourceKey: options.sourceKey,
+                subkey,
+                table: evaluated.table,
+                rowId: evaluated.id,
+                checksum: BigInt(checksum),
+                data: recordData
+            });
+            this.incrementBucket(definitionId, evaluated.bucket, op_id, byteEstimate);
+        }
+        for (let bucket of remaining_buckets.values()) {
+            const definitionId = bucket.definitionId;
+            const op_id = options.op_seq.next();
+            this.debugLastOpId = op_id;
+            this.addBucketDataRemove({
+                bucketKey: {
+                    replicationStreamId: this.group_id,
+                    definitionId,
+                    bucket: bucket.bucket
+                },
+                op_id,
+                sourceTableId: options.table.id,
+                sourceKey: options.sourceKey,
+                subkey,
+                table: bucket.table,
+                rowId: bucket.id,
+                checksum: dchecksum
+            });
+            this.currentSize += 200;
+            this.incrementBucket(definitionId, bucket.bucket, op_id, 200);
+        }
+    }
+    /**
+     * New bucket operations persist protocol subkeys during replication. Older
+     * V1 documents may not have one, so sync reads retain a fallback.
+     */
+    persistedSubkey(table, key) {
+        return replicaIdToSubkey(table, key);
+    }
+    get bucketDataCount() {
+        return this.bucketData.length;
+    }
+    incrementBucket(definitionId, bucket, op_id, bytes) {
+        const key = `${definitionId ?? ''}:${bucket}`;
+        let existingState = this.bucketStates.get(key);
+        if (existingState) {
+            existingState.lastOp = op_id;
+            existingState.incrementCount += 1;
+            existingState.incrementBytes += bytes;
+        }
+        else {
+            this.bucketStates.set(key, {
+                definitionId,
+                bucket,
+                lastOp: op_id,
+                incrementCount: 1,
+                incrementBytes: bytes,
+                incrementChunks: 0
+            });
+        }
+    }
+    /**
+     * V3 persists operations in chunks. Keep this separate from incrementBucket:
+     * operation counts are known while evaluating rows, while chunk counts and
+     * exact persisted bytes are only known after the writer has chunked a flush.
+     */
+    incrementBucketPersistedChunk(definitionId, bucket, bytes) {
+        const key = `${definitionId ?? ''}:${bucket}`;
+        const existingState = this.bucketStates.get(key);
+        if (existingState != null) {
+            existingState.incrementChunks += 1;
+            existingState.incrementBytes += bytes;
+        }
+    }
+    /**
+     * V3's compactor calculates byte deltas from persisted chunk metadata.
+     * Replace the writer's per-operation estimate with those exact sizes before
+     * flushing the corresponding bucket state.
+     */
+    resetBucketPersistedBytes(definitionId, bucket) {
+        const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
+        if (state != null) {
+            state.incrementBytes = 0;
+        }
+    }
+    addBucketDataPut(options) {
+        this.bucketData.push({
+            bucketKey: options.bucketKey,
+            o: options.op_id,
+            op: 'PUT',
+            source_table: mongoTableId(options.sourceTableId),
+            source_key: options.sourceKey,
+            subkey: options.subkey,
+            table: options.table,
+            row_id: options.rowId,
+            checksum: options.checksum,
+            data: options.data
+        });
+    }
+    addBucketDataRemove(options) {
+        this.bucketData.push({
+            bucketKey: options.bucketKey,
+            o: options.op_id,
+            op: 'REMOVE',
+            source_table: mongoTableId(options.sourceTableId),
+            source_key: options.sourceKey,
+            subkey: options.subkey,
+            table: options.table,
+            row_id: options.rowId,
+            checksum: options.checksum,
+            data: null
+        });
+    }
+    shouldFlushTransaction(currentDataCount) {
+        return (this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
+            this.bucketDataCount >= MAX_TRANSACTION_DOC_COUNT ||
+            currentDataCount >= MAX_TRANSACTION_DOC_COUNT ||
+            this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT);
+    }
+    /** Adopt one fully evaluated row; unfinished rows never mutate this buffer. */
+    append(row) {
+        // A single high-fanout row may exceed a normal batch target. Avoid the
+        // JavaScript argument-count limit while retaining its operation order.
+        for (const operation of row.bucketData) {
+            this.bucketData.push(operation);
+        }
+        for (const parameter of row.bucketParameters) {
+            this.bucketParameters.push(parameter);
+        }
+        for (const [key, state] of row.bucketStates) {
+            const previous = this.bucketStates.get(key);
+            this.bucketStates.set(key, {
+                ...state,
+                incrementCount: (previous?.incrementCount ?? 0) + state.incrementCount,
+                incrementBytes: (previous?.incrementBytes ?? 0) + state.incrementBytes,
+                incrementChunks: (previous?.incrementChunks ?? 0) + state.incrementChunks
+            });
+        }
+        this.currentSize += row.currentSize;
+        this.debugLastOpId = row.debugLastOpId ?? this.debugLastOpId;
+    }
+    clear() {
+        this.bucketData = [];
+        this.bucketParameters = [];
+        this.bucketStates.clear();
+        this.currentSize = 0;
+        this.debugLastOpId = null;
+    }
+    logFlush(startAt, currentDataCount, options) {
+        const flushedSomething = this.bucketDataCount > 0 ||
+            this.bucketParameters.length > 0 ||
+            currentDataCount > 0 ||
+            this.bucketStates.size > 0;
+        if (flushedSomething) {
+            const duration = Math.round(performance.now() - startAt);
+            if (options?.oldestUncommittedChange != null) {
+                const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
+                this.logger.info(`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${currentDataCount} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, {
+                    flushed: {
+                        duration: duration,
+                        size: this.currentSize,
+                        bucket_ops_count: this.bucketDataCount,
+                        parameter_indexes_count: this.bucketParameters.length,
+                        source_records_count: currentDataCount,
+                        replication_lag_seconds: replicationLag
+                    }
+                });
+            }
+            else {
+                this.logger.info(`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${currentDataCount} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, {
+                    flushed: {
+                        duration: duration,
+                        size: this.currentSize,
+                        bucket_ops_count: this.bucketDataCount,
+                        parameter_indexes_count: this.bucketParameters.length,
+                        source_records_count: currentDataCount
+                    }
+                });
+            }
+        }
+        const stats = {
+            bucketDataCount: this.bucketDataCount,
+            parameterDataCount: this.bucketParameters.length,
+            currentDataCount: currentDataCount,
+            flushedAny: flushedSomething
+        };
+        return stats;
+    }
+}
+//# sourceMappingURL=BatchOperationBuffer.js.map

package/dist/storage/implementation/common/BatchOperationBuffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BatchOperationBuffer.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/BatchOperationBuffer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAErD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAO9E;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAgBZ;IAEA;IAIA;IArBnB,UAAU,GAAoB,EAAE,CAAC;IACjC,gBAAgB,GAAoC,EAAE,CAAC;IACvD,YAAY,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEpD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,CAAS;IAEpB,YACmB,QAAgB,EACjC,WAAmB,EACF,WAGhB,EACgB,SAAiB,aAAa;QAN9B,aAAQ,GAAR,QAAQ,CAAQ;QAEhB,gBAAW,GAAX,WAAW,CAG3B;QACgB,WAAM,GAAN,MAAM,CAAwB;QAE/C,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAA8B;QAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAmC,CAAC;QACrE,KAAK,IAAI,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,MAAM,GAA4B;gBACtC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;gBACrE,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,IAAI,OAAO,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,YAAa,CAAC,EAAE,CAAC;gBAC9G,+CAA+C;gBAC/C,yGAAyG;gBACzG,SAAS;YACX,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnD,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC9E,MAAM,GAAG,GAAG,gBAAgB,CAAC;gBAC3B,YAAY,EAAE,YAAY;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,EAAE,EAAE,SAAS,CAAC,EAAE;aACjB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,MAAM,OAAO,GAAG,OAAO,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC;gBAC9E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC3B,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC;YAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,gBAAgB,CAAC;gBACpB,SAAS,EAAE;oBACT,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,YAAY,EAAE,YAAY;oBAC1B,mBAAmB,EAAE,IAAI,CAAC,QAAQ;iBACnC;gBACD,KAAK;gBACL,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,KAAK,EAAE,SAAS,CAAC,EAAE;gBACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC1B,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5E,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAa,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,mBAAmB,CAAC;gBACvB,SAAS,EAAE;oBACT,mBAAmB,EAAE,IAAI,CAAC,QAAQ;oBAClC,YAAY;oBACZ,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB;gBACD,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,KAA4B,EAAE,GAAsB;QAClE,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,eAAe,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAmB,EAAE,KAAa;QAClG,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;YAClC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzB,YAAY;gBACZ,MAAM;gBACN,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,KAAK;gBACrB,eAAe,EAAE,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,6BAA6B,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAa;QAC3F,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,aAAa,CAAC,eAAe,IAAI,CAAC,CAAC;YACnC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,yBAAyB,CAAC,YAAgC,EAAE,MAAc;QACxE,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,gBAAgB,CAAC,OAWhB;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,KAAK;YACT,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB,CAAC,OASnB;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,QAAQ;YACZ,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB,CAAC,gBAAwB;QAC7C,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,eAAe,IAAI,yBAAyB;YACjD,gBAAgB,IAAI,yBAAyB;YAC7C,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,MAAM,CAAC,GAAyB;QAC9B,uEAAuE;QACvE,uEAAuE;QACvE,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClC,CAAC;QACD,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC7C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzB,GAAG,KAAK;gBACR,cAAc,EAAE,CAAC,QAAQ,EAAE,cAAc,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,cAAc;gBACtE,cAAc,EAAE,CAAC,QAAQ,EAAE,cAAc,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,cAAc;gBACtE,eAAe,EAAE,CAAC,QAAQ,EAAE,eAAe,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,eAAe;aAC1E,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC;IAC/D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED,QAAQ,CAAC,OAAe,EAAE,gBAAwB,EAAE,OAA0C;QAC5F,MAAM,gBAAgB,GACpB,IAAI,CAAC,eAAe,GAAG,CAAC;YACxB,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;YAChC,gBAAgB,GAAG,CAAC;YACpB,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7B,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,EAAE,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAEnG,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,gBACF,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,sBAAsB,cAAc,GAAG,EAC7I;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,gBAAgB;wBACtC,uBAAuB,EAAE,cAAc;qBACxC;iBACF,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,gBACF,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,EAAE,EACxG;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,gBAAgB;qBACvC;iBACF,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAChD,gBAAgB,EAAE,gBAAgB;YAClC,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/storage/implementation/common/PersistedBatch.d.ts

@@ -1,16 +1,9 @@
-import { mongo } from '@powersync/lib-service-mongodb';
-import { BucketDataSource, BucketDefinitionId, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
 import { Logger } from '@powersync/lib-services-framework';
-import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
+import { InternalOpId, storage } from '@powersync/service-core';
+import { BucketDefinitionId, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import { MongoIdSequence } from '../MongoIdSequence.js';
-import { MongoWriteBatch } from '../MongoWriteBatch.js';
-import type { VersionedPowerSyncMongo } from '../db.js';
-import { TaggedBucketParameterDocument } from '../models.js';
-import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
-import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
 import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js';
-export declare const DEFAULT_INLINE_THRESHOLD_BYTES: number;
 export interface SaveBucketDataOptions {
     op_seq: MongoIdSequence;
     sourceKey: storage.ReplicaId;
@@ -34,102 +27,16 @@ export interface UpsertCurrentDataOptions {
 }
 export interface PersistedBatchOptions {
     logger?: Logger;
-    objectStorage?: ObjectStorage;
-    inlineThresholdBytes?: number;
-    /**
-     * Aborts in-flight object storage uploads when replication stops.
-     */
-    signal?: AbortSignal;
-    objectStorageUsageWriterId?: string;
 }
-/**
- * Keeps track of bulkwrite operations within a transaction.
- *
- * There may be multiple of these batches per transaction, but it may not span
- * multiple transactions.
- */
-export declare abstract class PersistedBatch {
-    protected readonly db: VersionedPowerSyncMongo;
-    protected readonly group_id: number;
-    protected readonly mapping: BucketDefinitionMapping;
-    logger: Logger;
-    bucketData: BucketDataDoc[];
-    bucketParameters: TaggedBucketParameterDocument[];
-    bucketStates: Map<string, BucketStateUpdate>;
-    protected readonly objectStorage?: ObjectStorage;
-    protected readonly inlineThresholdBytes: number;
-    protected readonly signal?: AbortSignal;
-    protected readonly objectStorageUsageWriterId?: string;
-    /**
-     * For debug logging only.
-     */
-    debugLastOpId: InternalOpId | null;
-    /**
-     * Very rough estimate of transaction size.
-     */
-    currentSize: number;
-    constructor(db: VersionedPowerSyncMongo, group_id: number, mapping: BucketDefinitionMapping, writtenSize: number, options?: PersistedBatchOptions);
+/** Operations used by the shared writer; each storage version owns its lifecycle. */
+export interface PersistedBatch {
+    readonly currentSize: number;
     saveBucketData(options: SaveBucketDataOptions): void;
-    abstract saveParameterData(data: SaveParameterDataOptions): void;
-    abstract hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void;
-    abstract softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, checkpointGreaterThan: bigint): void;
-    abstract upsertCurrentData(values: UpsertCurrentDataOptions): void;
-    protected abstract get currentDataCount(): number;
-    protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
-    protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
-    protected abstract queueCurrentData(writes: MongoWriteBatch): void;
-    protected abstract queueBucketStates(writes: MongoWriteBatch): void;
-    protected abstract resetCurrentData(): void;
-    protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
-    protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
-    /**
-     * New bucket operations persist protocol subkeys during replication. Older
-     * V1 documents may not have one, so sync reads retain a fallback.
-     */
-    protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string;
-    protected get bucketDataCount(): number;
-    protected incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number): void;
-    /**
-     * V3 persists operations in chunks. Keep this separate from incrementBucket:
-     * operation counts are known while evaluating rows, while chunk counts and
-     * exact persisted bytes are only known after the writer has chunked a flush.
-     */
-    protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number): void;
-    /**
-     * V3's compactor calculates byte deltas from persisted chunk metadata.
-     * Replace the writer's per-operation estimate with those exact sizes before
-     * flushing the corresponding bucket state.
-     */
-    protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string): void;
-    protected addBucketDataPut(options: {
-        op_id: InternalOpId;
-        bucketKey: BucketKey;
-        bucket: string;
-        sourceTableId: storage.SourceTable['id'];
-        sourceKey: storage.ReplicaId;
-        subkey: string;
-        table: string;
-        rowId: string;
-        checksum: bigint;
-        data: string;
-    }): void;
-    protected addBucketDataRemove(options: {
-        op_id: InternalOpId;
-        bucketKey: BucketKey;
-        sourceTableId: storage.SourceTable['id'];
-        sourceKey: storage.ReplicaId;
-        subkey: string;
-        table: string;
-        rowId: string;
-        checksum: bigint;
-    }): void;
+    saveParameterData(options: SaveParameterDataOptions): void;
+    hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void;
+    softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, checkpointGreaterThan: bigint): void;
+    upsertCurrentData(values: UpsertCurrentDataOptions): void;
     shouldFlushTransaction(): boolean;
-    flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions): Promise<{
-        bucketDataCount: number;
-        parameterDataCount: number;
-        currentDataCount: number;
-        flushedAny: boolean;
-    }>;
 }
 export interface BucketStateUpdate {
     definitionId: BucketDefinitionId | null;