@powersync/service-module-mongodb-storage 0.19.0 → 0.21.0

package/dist/storage/implementation/v3/MongoCompactorV3.js

@@ -1,169 +1,557 @@
-import { mongo } from '@powersync/lib-service-mongodb';
-import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
-import { addChecksums, utils } from '@powersync/service-core';
+import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { addChecksums, formatBytes, utils } from '@powersync/service-core';
 import { ConcurrentCompactionError, MongoCompactor } from '../MongoCompactor.js';
 import { cacheKey } from '../OperationBatch.js';
 import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js';
 import { BucketDataContextV3 } from './BucketDataContextV3.js';
 import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js';
+import { applyStatsReplacement, bucketStats, chooseCompactionKind, combineAdjacentStats, combineChunkStats, CompactionContext, CompactionKind, emptyBucketStats, firstUncompactedWrite, forcedCompactionKind, readCompactionBatch, statsForDocument, statsForDocuments, unclaimedSnapshotFilter } from './compact-utils.js';
+import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js';
+import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js';
 import { BucketDataObjectStorage, hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
 import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
-/**
- * Read one bounded prefix from a descending compaction cursor.
- *
- * The document that would cross the byte limit is deliberately not returned:
- * pagination resumes below the last returned `_id`, so that document remains
- * eligible for the next query. The first document is always accepted to ensure
- * progress when a single document exceeds the configured byte limit.
- *
- * `hasMore` is conservative when the document limit is reached. An extra empty
- * query is preferable to exhausting the cursor just to determine whether the
- * limited MongoDB query contained another document.
- */
-async function readCompactionBatch(cursor, options) {
-    const documents = [];
-    let cumulativeBytes = 0;
-    try {
-        for await (const document of cursor) {
-            if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) {
-                return { documents, hasMore: true };
-            }
-            documents.push(document);
-            cumulativeBytes += document.size;
-            if (documents.length >= options.documentLimit) {
-                return { documents, hasMore: true };
-            }
-        }
-        return { documents, hasMore: false };
-    }
-    finally {
-        await cursor.close();
-    }
-}
+import { createObjectStorageUsageWriterId, ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
+const DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS = 2 * 60 * 60 * 1000;
+const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000;
+const SCHEDULED_COMPACTION_BATCH_SIZE = 100;
 export class MongoCompactorV3 extends MongoCompactor {
+    minCompactChunkIntervalMs;
+    minCompactFullIntervalMs;
+    maxCompactFullIntervalMs;
+    compactLeaseDurationMs;
+    maxOpIdCap;
+    objectStorageUsage;
+    constructor(bucketStorage, db, options) {
+        super(bucketStorage, db, options);
+        this.minCompactChunkIntervalMs = options.minCompactChunkIntervalMs ?? DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS;
+        this.minCompactFullIntervalMs = options.minCompactFullIntervalMs ?? DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS;
+        this.maxCompactFullIntervalMs = options.maxCompactFullIntervalMs ?? DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS;
+        this.compactLeaseDurationMs = options.compactLeaseDurationMs ?? DEFAULT_COMPACT_LEASE_DURATION_MS;
+        this.maxOpIdCap = options.maxOpId;
+        this.objectStorageUsage = new ObjectStorageUsage(this.db, this.group_id, createObjectStorageUsageWriterId());
+    }
     async compact() {
         if (this.storage.objectStorage) {
             // Clean these before compacting - should be quick in most cases.
             try {
-                await this.objectStorageLifecycle.cleanup(this.logger);
+                await this.objectStorageLifecycle.cleanup(this.logger, { signal: this.signal });
             }
             catch (e) {
+                if (this.signal?.aborted) {
+                    throw e;
+                }
                 // In this case, still continue normal compact process
                 this.logger.error(`Failed to clean up object storage deletion markers before compaction`, e);
             }
         }
-        await super.compact();
+        await this.deleteOldCheckpointRequests();
+        if (this.buckets != null) {
+            await this.compactExplicitBuckets(this.buckets);
+        }
+        else if (this.compactChunksOnly) {
+            // Writers defer their first chunk-compaction check by this fixed default.
+            // Include that interval so this synchronous initial-replication pass
+            // processes the work that existed when it started.
+            await this.compactScheduledBuckets({
+                dueAheadMs: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS,
+                forceKind: CompactionKind.Chunks
+            });
+        }
+        else {
+            await this.compactScheduledBuckets();
+        }
         if (this.storage.objectStorage) {
             // Cleanup for any produced during compacting.
             // Note that markers only expire after a delay, so this may skip many produced during this compact
             // run. However, during long compact runs, this may also have many ones it can clean up.
-            await this.objectStorageLifecycle.cleanup(this.logger);
+            await this.objectStorageLifecycle.cleanup(this.logger, { signal: this.signal });
+            await this.objectStorageUsage.foldStaleWriterDeltas();
         }
+        return this.compactedBucketCount;
     }
-    get objectStorageLifecycle() {
-        if (!this.storage.objectStorage) {
-            throw new Error('Object storage is not configured');
+    async deleteOldCustomCheckpointRequests() {
+        if (this.deleteCheckpointRequestsBefore == null) {
+            return;
+        }
+        for (const collection of await this.db.listCustomCheckpointRequestCollections(this.group_id)) {
+            this.signal?.throwIfAborted();
+            await collection.deleteMany({
+                checkpoint_requested_at: { $exists: true, $lt: this.deleteCheckpointRequestsBefore }
+            });
         }
-        return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage);
     }
-    async *dirtyBucketBatches(options) {
-        if (options.minBucketChanges <= 0) {
-            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    /** An explicit compact request always runs a full compact for its buckets. */
+    async compactExplicitBuckets(buckets) {
+        for (const bucket of buckets) {
+            // This is not a super efficient query, but this is not a common use case.
+            // May be optimized later.
+            const states = await this.db
+                .bucketState(this.group_id)
+                .find({ '_id.b': bucket }, { projection: { _id: 1 } })
+                .toArray();
+            for (const state of states) {
+                await using lease = await this.claimBucket({ _id: state._id });
+                if (lease == null || lease.state.first_uncompacted_write == null) {
+                    continue;
+                }
+                if (this.isCompactionTargetCovered(lease.state, CompactionKind.Full)) {
+                    continue;
+                }
+                const decision = chooseCompactionKind(lease.state, lease.startedAt, this);
+                await this.compactClaimedBucket(lease, CompactionKind.Full, decision);
+            }
         }
-        const collection = this.db.bucketState(this.group_id);
-        yield* this.dirtyBucketBatchesForCollection(collection, { d: new mongo.MinKey(), b: new mongo.MinKey() }, { d: new mongo.MaxKey(), b: new mongo.MaxKey() }, options, (bucketState) => bucketState._id.d);
     }
-    async dirtyBucketBatchForChecksums(options) {
-        if (options.minBucketChanges <= 0) {
-            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    /**
+     * Process scheduled work in bounded batches.
+     *
+     * Batching specifically help to cover cases of many buckets where no compaction is required:
+     * Instead of sequentially claiming and then rescheduling a bucket, this handles it in bulk.
+     *
+     * Buckets that do need compaction are still claimed and processed sequentially.
+     *
+     * Any concurrent workers may read the same batch. Rescheduling filters out buckets handled
+     * by a concurrent worker or replication write, while buckets that do need compaction are
+     * filtered out when claiming a compaction lease.
+     *
+     * We filter scheduled jobs by the job start date, so that the same bucket is not compacted
+     * multiple times in one run. Reschedules fall beyond the fixed boundary. For the run after
+     * initial replication, dueAheadMs extends that boundary to include the first deferred interval.
+     */
+    async compactScheduledBuckets(options = {}) {
+        // Writers derive next_compact_check from MongoDB's $$NOW. Use the same
+        // clock for the fixed job boundary so clock skew cannot exclude work at
+        // the exact initial-replication interval.
+        const [{ now: jobStartedAt }] = await this.db.db
+            .aggregate([{ $documents: [{}] }, { $project: { _id: 0, now: '$$NOW' } }])
+            .toArray();
+        const dueBefore = new Date(jobStartedAt.getTime() + (options.dueAheadMs ?? 0));
+        const forceKind = options.forceKind;
+        const rescheduleNotBefore = new Date(dueBefore.getTime() + 1);
+        while (true) {
+            this.signal?.throwIfAborted();
+            const states = await this.findScheduledBucketBatch(dueBefore);
+            if (states.length == 0) {
+                break;
+            }
+            const scheduled = [];
+            for (const state of states) {
+                try {
+                    scheduled.push({
+                        state,
+                        decision: chooseCompactionKind(state, jobStartedAt, this),
+                        forcedKind: forcedCompactionKind(state, forceKind, this)
+                    });
+                }
+                catch (error) {
+                    await this.rescheduleFailedBucket(state, rescheduleNotBefore, error);
+                }
+            }
+            const noOpStates = scheduled.filter(({ state, decision, forcedKind }) => state.compact_lease == null && (forceKind == null ? decision.kind : forcedKind) == null);
+            await this.rescheduleUnclaimedBuckets(noOpStates, rescheduleNotBefore);
+            for (const { state, decision, forcedKind } of scheduled) {
+                const kind = forceKind == null ? decision.kind : forcedKind;
+                if (state.compact_lease == null && kind == null) {
+                    continue;
+                }
+                try {
+                    await using lease = await this.claimBucket({ _id: state._id, next_compact_check: { $lte: dueBefore } });
+                    if (lease == null) {
+                        continue;
+                    }
+                    const claimedDecision = chooseCompactionKind(lease.state, lease.startedAt, this);
+                    const claimedKind = forceKind == null ? claimedDecision.kind : forcedCompactionKind(lease.state, forceKind, this);
+                    if (claimedKind == null) {
+                        await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore);
+                    }
+                    else if (this.isCompactionTargetCovered(lease.state, claimedKind)) {
+                        // The run cannot advance this kind's watermark without regressing
+                        // already-published progress. Keep any newer work scheduled.
+                        await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore);
+                    }
+                    else {
+                        await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore);
+                    }
+                }
+                catch (error) {
+                    if (this.signal?.aborted) {
+                        // When aborted, stop completely, rather than logging and re-scheduling individual buckets.
+                        // The lease on the current bucket is still released automatically.
+                        throw error;
+                    }
+                    await this.rescheduleFailedBucket(state, rescheduleNotBefore, error);
+                }
+            }
         }
-        return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketState(this.group_id), {
-            'estimate_since_compact.count': { $gte: options.minBucketChanges }
-        }, (bucketState) => bucketState._id.d);
     }
-    async writeBucketStateUpdates() {
-        await this.db
+    /** Read a bounded, priority-ordered snapshot of currently claimable scheduled work. */
+    async findScheduledBucketBatch(dueBefore) {
+        return this.db
             .bucketState(this.group_id)
-            .bulkWrite(this.bucketStateUpdates, {
-            ordered: false
-        });
+            .find({
+            next_compact_check: { $lte: dueBefore },
+            ...AVAILABLE_LEASE_EXPR
+        })
+            .sort({ next_compact_check: 1 })
+            .limit(SCHEDULED_COMPACTION_BATCH_SIZE)
+            .toArray();
     }
     /**
-     * The compactor operates on persisted definition ids only - never on parsed sources.
-     * This narrowed view makes the source-resolving checksum methods unreachable here.
+     * Reschedule snapshots that were already known to be no-ops without first
+     * taking a lease. Every decision input is compared so a concurrent writer
+     * or compactor simply makes the update a no-op instead of losing work. A
+     * successful reschedule moves beyond this run's fixed selection boundary.
      */
-    get definitionChecksums() {
-        return this.storage.checksums;
-    }
-    async computeChecksumsForBuckets(buckets) {
-        return this.definitionChecksums.computePartialChecksumsDirectByDefinition(buckets.map(({ bucket, definitionId }) => {
-            if (definitionId == null) {
-                throw new ServiceAssertionError(`Missing definitionId for bucket checksum update on bucket ${bucket}`);
+    async rescheduleUnclaimedBuckets(states, notBefore) {
+        if (states.length == 0) {
+            return;
+        }
+        await this.db.bucketState(this.group_id).bulkWrite(states.map(({ state, decision }) => ({
+            updateOne: {
+                filter: unclaimedSnapshotFilter(state),
+                update: [{ $set: { next_compact_check: this.rescheduleAtOrAfter(decision.nextCompactCheck, notBefore) } }]
             }
-            return {
-                bucket,
-                definitionId,
-                end: this.maxOpId
-            };
-        }));
+        })), { ordered: false });
     }
-    bucketStateFilter(bucket, definitionId) {
-        if (definitionId == null) {
-            throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`);
+    /**
+     * Isolate a malformed bucket so it cannot prevent other scheduled buckets
+     * from compacting. The snapshot filter preserves any concurrent write or
+     * compactor result instead of overwriting its next check.
+     */
+    async rescheduleFailedBucket(state, notBefore, error) {
+        this.logger.error(`Failed to compact scheduled bucket ${state._id.b}; rescheduling it`, error);
+        try {
+            await this.db
+                .bucketState(this.group_id)
+                .updateOne(unclaimedSnapshotFilter(state), [{ $set: { next_compact_check: notBefore } }]);
         }
-        return {
-            _id: {
-                d: definitionId,
-                b: bucket
+        catch (rescheduleError) {
+            this.logger.error(`Failed to reschedule bucket ${state._id.b} after a compaction error`, rescheduleError);
+        }
+    }
+    /**
+     * Given a bucket filter, claim a lease on the bucket. The filter should include a filter on _id.
+     *
+     * Resolves to null if the bucket is already claimed, not found, or filtered out.
+     */
+    async claimBucket(filter, sort) {
+        return CompactionLease.claim(this.db.bucketState(this.group_id), filter, sort, this.compactLeaseDurationMs);
+    }
+    async compactClaimedBucket(lease, kind, decision, rescheduleNotBefore) {
+        const context = new CompactionContext(lease, kind, decision, rescheduleNotBefore, this.compactionTarget(lease.state));
+        lease.startRenewal();
+        await this.compactSingleBucket(context);
+    }
+    async rescheduleClaimedBucket(lease, decision, notBefore) {
+        await lease.reschedule(this.rescheduleAtOrAfter(decision.nextCompactCheck, notBefore));
+    }
+    rescheduleAtOrAfter(nextCompactCheck, notBefore) {
+        return notBefore == null ? nextCompactCheck : { $max: [nextCompactCheck, notBefore] };
+    }
+    compactionTarget(state) {
+        return this.maxOpIdCap == null || state.last_op < this.maxOpIdCap ? state.last_op : this.maxOpIdCap;
+    }
+    isCompactionTargetCovered(state, kind) {
+        const target = this.compactionTarget(state);
+        if (kind == CompactionKind.Chunks) {
+            return state.compacted_state != null && state.compacted_state.op_id >= target;
+        }
+        if (state.last_full_compact != null && state.last_full_compact.op_id >= target) {
+            return true;
+        }
+        // A full compact may change counts before the checksum-cache boundary.
+        // Wait for the safe target to catch up instead of publishing an older or
+        // stale cache. At the same boundary, full coverage can still advance.
+        return state.compacted_state != null && state.compacted_state.op_id > target;
+    }
+    get objectStorageLifecycle() {
+        if (!this.storage.objectStorage) {
+            throw new Error('Object storage is not configured');
+        }
+        return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage);
+    }
+    async compactSingleBucket(context) {
+        if (context.kind == CompactionKind.Chunks) {
+            return this.compactSingleBucketChunks(context);
+        }
+        return this.compactSingleBucketFully(context);
+    }
+    /**
+     * Merge adjacent bucket-data chunks without inspecting their operations
+     * unless a merge is possible. The metadata contains enough information to
+     * update the persisted checksum state and to decide whether a group can fit
+     * in one chunk.
+     */
+    async compactSingleBucketChunks(context) {
+        const bucket = context.state._id.b;
+        const resolvedDefinitionId = context.state._id.d;
+        const bucketContext = new BucketDataContextV3(this.db, {
+            bucket,
+            definitionId: resolvedDefinitionId,
+            replicationStreamId: this.group_id
+        });
+        const collection = this.db.bucketData(this.group_id, resolvedDefinitionId);
+        const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId };
+        let previousCompactedState = context.state.compacted_state;
+        // A zero boundary represents an empty prefix, so there is no stored chunk
+        // whose statistics need to be carried into this pass.
+        if (previousCompactedState?.op_id === 0n) {
+            previousCompactedState = undefined;
+        }
+        // Include the last previously compacted chunk as well as new chunks. It
+        // is the only old chunk which can become mergeable with the new tail.
+        let lowerBound = previousCompactedState != null ? bucketContext.docId(previousCompactedState.op_id - 1n) : bucketContext.minId;
+        const upperBound = bucketContext.docId(context.targetOp + 1n);
+        let cachedBoundaryToVerify = previousCompactedState?.op_id;
+        let compactedOpId = null;
+        let overlappingCompactedChunk;
+        let compactedTail = emptyBucketStats();
+        let pendingChunks = [];
+        let pendingSize = 0;
+        while (true) {
+            this.signal?.throwIfAborted();
+            await context.lease.throwIfLost();
+            const batch = await readCompactionBatch(collection.aggregate([
+                {
+                    $match: {
+                        _id: {
+                            $gt: lowerBound,
+                            $lt: upperBound
+                        }
+                    }
+                },
+                { $sort: { _id: 1 } },
+                { $limit: this.moveBatchQueryLimit },
+                {
+                    $project: {
+                        _id: 1,
+                        min_op: 1,
+                        checksum: 1,
+                        count: 1,
+                        size: 1,
+                        target_op: 1,
+                        storage_ref: 1
+                    }
+                }
+            ], { batchSize: this.moveBatchQueryLimit + 1 }), {
+                byteLimit: this.moveBatchByteLimit,
+                documentLimit: this.moveBatchQueryLimit
+            });
+            if (cachedBoundaryToVerify != null) {
+                const cachedBoundary = cachedBoundaryToVerify;
+                cachedBoundaryToVerify = undefined;
+                if (batch.documents[0]?._id.o !== cachedBoundary) {
+                    // A previous attempt may have replaced the cached boundary before
+                    // finalizing bucket state. Keep the persisted cache available to
+                    // readers, but ignore it in this attempt and calculate its
+                    // replacement through the normal scan from the bucket beginning.
+                    previousCompactedState = undefined;
+                    lowerBound = bucketContext.minId;
+                    continue;
+                }
+            }
+            if (batch.documents.length == 0) {
+                break;
+            }
+            for (const doc of batch.documents) {
+                compactedOpId = maxOpId(compactedOpId, doc._id.o);
+                const documentStats = statsForDocument(doc);
+                if (previousCompactedState?.op_id === doc._id.o) {
+                    overlappingCompactedChunk = documentStats;
+                }
+                const nextSize = pendingSize + doc.size;
+                if (pendingChunks.length > 0 && nextSize > DEFAULT_MAX_DOC_SIZE_BYTES) {
+                    const groupStats = await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext);
+                    compactedTail = combineAdjacentStats(compactedTail, groupStats);
+                    pendingChunks = [];
+                    pendingSize = 0;
+                }
+                pendingChunks.push(doc);
+                pendingSize += doc.size;
             }
+            lowerBound = batch.documents[batch.documents.length - 1]._id;
+            if (!batch.hasMore) {
+                break;
+            }
+        }
+        if (pendingChunks.length > 0) {
+            const groupStats = await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext);
+            compactedTail = combineAdjacentStats(compactedTail, groupStats);
+        }
+        if (compactedOpId == null) {
+            await this.finalizeSkippedBucket(context);
+            return;
+        }
+        const compactedState = previousCompactedState == null
+            ? compactedTail
+            : combineChunkStats(previousCompactedState, compactedTail, overlappingCompactedChunk);
+        const tailStats = compactedOpId == context.lastOp
+            ? undefined
+            : await this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp, bucketContext.docId(compactedOpId));
+        const result = {
+            compactedState,
+            bucketStats: tailStats == null ? compactedState : combineAdjacentStats(compactedState, tailStats)
         };
+        await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: 0 });
+        this.compactedBucketCount++;
+        this.logger.info(`Compacted bucket chunks ${bucket}: ${result.bucketStats.count} ops, ${result.bucketStats.chunks} chunks, ${formatBytes(result.bucketStats.bytes)}`);
     }
-    async getBucketDataContext(bucket, definitionId) {
-        let resolvedDefinitionId = definitionId;
-        if (resolvedDefinitionId == null) {
-            const allDefinitionIds = this.storage.storageIds.bucketDefinitionIds;
-            if (allDefinitionIds.length > 0) {
-                const potentialIds = allDefinitionIds.map((id) => ({ d: id, b: bucket }));
-                const bucketState = await this.db.bucketState(this.group_id).findOne({
-                    _id: { $in: potentialIds }
-                });
-                if (bucketState != null) {
-                    resolvedDefinitionId = bucketState._id.d;
-                }
+    async flushChunkMerge(bucket, inputs, collection, context, bucketContext) {
+        if (inputs.length == 1) {
+            return statsForDocument(inputs[0]);
+        }
+        // The metadata scan deliberately excluded ops. Read inline payloads only
+        // for this merge group; object-storage payloads are fetched below using
+        // the same rule.
+        const inlineInputs = inputs.filter((input) => input.storage_ref == null);
+        if (inlineInputs.length > 0) {
+            const inlineDocuments = await collection
+                .find({ _id: { $in: inlineInputs.map((input) => input._id) } }, { projection: { _id: 1, ops: 1 } })
+                .toArray();
+            const opsById = new Map(inlineDocuments.map((document) => [document._id.o.toString(), document.ops]));
+            for (const input of inlineInputs) {
+                input.ops = opsById.get(input._id.o.toString());
             }
         }
-        if (resolvedDefinitionId == null) {
-            return null;
+        await hydrateBucketDataDocuments(inputs, this.storage.objectStorage, { signal: this.signal });
+        const operations = inputs.flatMap((input) => Array.from(loadBucketDataDocument(context, input)));
+        const targetOp = inputs.reduce((maxTarget, input) => maxOpId(maxTarget, input.target_op), null);
+        const result = await this.flushCompactionGroup(bucket, {
+            inputs,
+            ops: operations,
+            changed: true,
+            targetOp
+        }, bucketContext, context);
+        return result.stats;
+    }
+    async finalizeCompactedBucket({ context, compactedOpId, compactionResult, puts }) {
+        await context.lease.throwIfLost();
+        const startedStats = bucketStats(context.state);
+        const delta = {
+            count: compactionResult.bucketStats.count - startedStats.count,
+            bytes: compactionResult.bucketStats.bytes - startedStats.bytes,
+            chunks: compactionResult.bucketStats.chunks - startedStats.chunks
+        };
+        const coveredClaimedHead = compactedOpId >= context.lastOp;
+        const concurrentWriteCheck = { $gt: ['$last_op', context.lastOp] };
+        const remainingFullWorkCheck = coveredClaimedHead ? concurrentWriteCheck : true;
+        const nextAfterPartialFullCompact = this.rescheduleAtOrAfter({ $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } }, context.rescheduleNotBefore);
+        const nextCheckForUncompactedWork = this.rescheduleAtOrAfter({
+            $min: [
+                new Date(firstUncompactedWrite(context.state).getTime() + this.maxCompactFullIntervalMs),
+                { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } }
+            ]
+        }, context.rescheduleNotBefore);
+        const update = {
+            compacted_state: {
+                op_id: compactedOpId,
+                checksum: BigInt(compactionResult.compactedState.checksum),
+                count: compactionResult.compactedState.count,
+                bytes: compactionResult.compactedState.bytes,
+                chunks: compactionResult.compactedState.chunks,
+                at: '$$NOW'
+            },
+            bucket_stats: {
+                count: { $add: ['$bucket_stats.count', delta.count] },
+                bytes: { $add: ['$bucket_stats.bytes', delta.bytes] },
+                chunks: { $add: ['$bucket_stats.chunks', delta.chunks] }
+            },
+            first_uncompacted_write: context.kind == CompactionKind.Full
+                ? { $cond: [remainingFullWorkCheck, '$$NOW', '$$REMOVE'] }
+                : '$first_uncompacted_write',
+            next_compact_check: context.kind == CompactionKind.Full
+                ? { $cond: [remainingFullWorkCheck, nextAfterPartialFullCompact, '$$REMOVE'] }
+                : nextCheckForUncompactedWork
+        };
+        if (context.kind == CompactionKind.Full) {
+            update.last_full_compact = {
+                op_id: compactedOpId,
+                count: compactionResult.compactedState.count,
+                puts,
+                at: '$$NOW'
+            };
         }
-        return new BucketDataContextV3(this.db, {
+        await context.lease.finalize(update);
+    }
+    async finalizeSkippedBucket(context) {
+        // A maxOpId cap can exclude the first remaining document entirely. Avoid
+        // immediately claiming the same no-progress bucket again in this run.
+        await this.rescheduleClaimedBucket(context.lease, {
+            ...context.decision,
+            nextCompactCheck: {
+                $max: [
+                    context.decision.nextCompactCheck,
+                    { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } }
+                ]
+            }
+        }, context.rescheduleNotBefore);
+    }
+    /**
+     * Read bucket stats directly from bucket_data documents.
+     */
+    async readBucketStats(bucket, definitionId, maxOp, lowerBound) {
+        const context = new BucketDataContextV3(this.db, {
             bucket,
-            definitionId: resolvedDefinitionId,
+            definitionId,
             replicationStreamId: this.group_id
         });
+        const [stats] = await this.db
+            .bucketData(this.group_id, definitionId)
+            .aggregate([
+            {
+                $match: {
+                    _id: lowerBound == null
+                        ? { $gte: context.minId, $lte: context.docId(maxOp) }
+                        : { $gt: lowerBound, $lte: context.docId(maxOp) }
+                }
+            },
+            {
+                $group: {
+                    _id: null,
+                    count: { $sum: '$count' },
+                    bytes: { $sum: '$size' },
+                    chunks: { $sum: 1 },
+                    checksum: { $sum: '$checksum' }
+                }
+            }
+        ])
+            .toArray();
+        return {
+            count: Number(stats?.count ?? 0),
+            bytes: BigInt(stats?.bytes ?? 0),
+            chunks: Number(stats?.chunks ?? 0),
+            checksum: typeof stats?.checksum == 'bigint'
+                ? Number(BigInt.asIntN(32, stats.checksum))
+                : addChecksums(0, Number(stats?.checksum ?? 0))
+        };
     }
-    async compactSingleBucket(bucket, definitionId = null) {
-        const bucketContext = await this.getBucketDataContext(bucket, definitionId);
-        if (bucketContext == null) {
-            return;
-        }
-        const resolvedDefinitionId = bucketContext.key.definitionId;
+    async compactSingleBucketFully(context) {
+        const bucket = context.state._id.b;
+        const resolvedDefinitionId = context.state._id.d;
+        const bucketContext = new BucketDataContextV3(this.db, {
+            bucket,
+            definitionId: resolvedDefinitionId,
+            replicationStreamId: this.group_id
+        });
         const collection = this.db.bucketData(this.group_id, resolvedDefinitionId);
-        const context = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId };
+        const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId };
         const lowerBound = bucketContext.minId;
-        let upperBound = bucketContext.docId(this.maxOpId + 1n);
-        let totalChecksum = 0;
+        let upperBound = bucketContext.docId(context.targetOp + 1n);
         let totalOpCount = 0;
-        let totalOpBytes = 0;
         let lastNotPut = null;
         let opsSincePut = 0;
         let compactedOpId = null;
         let clearBoundary = null;
+        let compactedStats = emptyBucketStats();
         const seen = new Map();
         let trackingSize = 0;
+        let putCount = 0;
         let pendingGroup = null;
         // --- Read batch from MongoDB ---
         while (true) {
             this.signal?.throwIfAborted();
+            await context.lease.throwIfLost();
             const pipeline = [
                 {
                     $match: {
@@ -205,7 +593,7 @@ export class MongoCompactorV3 extends MongoCompactor {
             // merging is useful, and writes each final object at most once.
             for (const doc of batchDocs) {
                 compactedOpId ??= doc._id.o;
-                const originalOps = Array.from(loadBucketDataDocument(context, doc));
+                const originalOps = Array.from(loadBucketDataDocument(dataContext, doc));
                 let changed = false;
                 const compactedOps = [];
                 let maxTargetOp = doc.target_op ?? null;
@@ -223,6 +611,7 @@ export class MongoCompactorV3 extends MongoCompactor {
                                 row_id: undefined,
                                 source_table: undefined,
                                 source_key: undefined,
+                                subkey: undefined,
                                 data: null
                             });
                             changed = true;
@@ -238,6 +627,7 @@ export class MongoCompactorV3 extends MongoCompactor {
                             }
                             compactedOps.push(op);
                             if (op.op == 'PUT') {
+                                putCount++;
                                 lastNotPut = null;
                                 opsSincePut = 0;
                             }
@@ -260,10 +650,6 @@ export class MongoCompactorV3 extends MongoCompactor {
                     }
                 }
                 compactedOps.reverse();
-                for (const op of compactedOps) {
-                    totalChecksum = addChecksums(totalChecksum, Number(op.checksum));
-                    totalOpBytes += op.data?.length ?? 0;
-                }
                 totalOpCount += compactedOps.length;
                 const candidate = {
                     inputs: [doc],
@@ -287,11 +673,12 @@ export class MongoCompactorV3 extends MongoCompactor {
                     }
                     else {
                         const flushedGroup = pendingGroup;
-                        const documentId = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, context);
+                        const result = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, dataContext);
+                        compactedStats = combineAdjacentStats(compactedStats, result.stats);
                         if (lastNotPut != null &&
                             flushedGroup.ops[0].o <= lastNotPut &&
                             flushedGroup.ops[flushedGroup.ops.length - 1].o >= lastNotPut) {
-                            clearBoundary = { opId: lastNotPut, documentId };
+                            clearBoundary = { opId: lastNotPut, documentId: result.documentId };
                         }
                         pendingGroup = candidate;
                     }
@@ -305,14 +692,16 @@ export class MongoCompactorV3 extends MongoCompactor {
             this.logger.info(`Compacted batch of ${batchDocs.length} documents for bucket ${bucket}`);
         }
         if (pendingGroup != null) {
-            const documentId = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, context);
+            const result = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, dataContext);
+            compactedStats = combineAdjacentStats(compactedStats, result.stats);
             if (lastNotPut != null &&
                 pendingGroup.ops[0].o <= lastNotPut &&
                 pendingGroup.ops[pendingGroup.ops.length - 1].o >= lastNotPut) {
-                clearBoundary = { opId: lastNotPut, documentId };
+                clearBoundary = { opId: lastNotPut, documentId: result.documentId };
             }
         }
         if (compactedOpId == null) {
+            await this.finalizeSkippedBucket(context);
             return;
         }
         // --- Clear: collapse leading MOVE/REMOVE/CLEAR sequence ---
@@ -320,25 +709,21 @@ export class MongoCompactorV3 extends MongoCompactor {
             if (clearBoundary == null || clearBoundary.opId != lastNotPut) {
                 throw new ReplicationAssertionError(`Missing CLEAR boundary document for bucket ${bucket}`);
             }
-            totalOpCount += await this.clearBucketLeading(lastNotPut, clearBoundary.documentId, bucketContext, collection, context);
+            const clearResult = await this.clearBucketLeading(lastNotPut, clearBoundary.documentId, bucketContext, collection, dataContext);
+            totalOpCount += clearResult.opCountDiff;
+            compactedStats = applyStatsReplacement(compactedStats, clearResult.before, clearResult.after);
         }
+        const tailStats = compactedOpId == context.lastOp
+            ? undefined
+            : await this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp, bucketContext.docId(compactedOpId));
+        const result = {
+            compactedState: compactedStats,
+            bucketStats: tailStats == null ? compactedStats : combineAdjacentStats(compactedStats, tailStats)
+        };
         // --- Finalize: update bucket checksums and state ---
-        this.updateBucketChecksums({
-            bucket,
-            definitionId: resolvedDefinitionId,
-            seen: new Map(),
-            trackingSize: 0,
-            lastNotPut: lastNotPut,
-            opsSincePut: opsSincePut,
-            checksum: totalChecksum,
-            opCount: totalOpCount,
-            opBytes: totalOpBytes
-        }, compactedOpId);
-        if (this.bucketStateUpdates.length > 0) {
-            await this.writeBucketStateUpdates();
-            this.bucketStateUpdates = [];
-        }
-        logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`);
+        await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: putCount });
+        this.compactedBucketCount++;
+        this.logger.info(`Compacted bucket ${bucket}: ${totalOpCount} ops, ${result.bucketStats.chunks} chunks, ${formatBytes(result.bucketStats.bytes)}`);
     }
     /**
      * Persist replacement objects before starting the transaction, then atomically
@@ -348,7 +733,10 @@ export class MongoCompactorV3 extends MongoCompactor {
      */
     async flushCompactionGroup(bucket, group, bucketContext, context) {
         if (group.inputs.length == 1 && !group.changed) {
-            return group.inputs[0]._id;
+            return {
+                documentId: group.inputs[0]._id,
+                stats: statsForDocument(group.inputs[0])
+            };
         }
         const inputs = group.inputs;
         const idsToDelete = inputs.map((doc) => doc._id);
@@ -356,6 +744,7 @@ export class MongoCompactorV3 extends MongoCompactor {
         const expectedChecksum = inputs.reduce((sum, doc) => sum + doc.checksum, 0n);
         const expectedOpCount = inputs.reduce((sum, doc) => sum + doc.count, 0);
         const oldStoragePaths = inputs.flatMap((doc) => (doc.storage_ref ? [doc.storage_ref.path] : []));
+        const oldStorageBytes = inputs.reduce((sum, document) => sum + ObjectStorageUsage.bytes(document), 0n);
         const { documents, storagePaths: newStoragePaths, uploads } = await this.persistBucketData(bucket, [group.ops], context, undefined, { targetOp: group.targetOp });
         const session = this.db.client.startSession();
         try {
@@ -382,6 +771,7 @@ export class MongoCompactorV3 extends MongoCompactor {
                 await bucketContext.collection.deleteMany({ _id: { $in: idsToDelete } }, { session });
                 await bucketContext.collection.insertMany(documents, { session });
                 await this.finishObjectStorageReplacement(oldStoragePaths, newStoragePaths, uploads, session);
+                await this.recordObjectStorageReplacement(oldStorageBytes, documents, context.definitionId, session);
             }, {
                 writeConcern: { w: 'majority' },
                 readConcern: { level: 'snapshot' }
@@ -390,7 +780,10 @@ export class MongoCompactorV3 extends MongoCompactor {
         finally {
             await session.endSession();
         }
-        return documents[0]._id;
+        return {
+            documentId: documents[0]._id,
+            stats: statsForDocuments(documents)
+        };
     }
     /**
      * Collapse the leading sequence of MOVE/REMOVE/CLEAR ops at the start
@@ -398,10 +791,13 @@ export class MongoCompactorV3 extends MongoCompactor {
      * before the known boundary document, then splits that boundary document
      * if it contains ops on both sides of lastNotPut.
      *
-     * Returns the op count diff after replacing cleared ops with CLEAR ops.
+     * Returns the op count and stored-stat changes after replacing cleared ops
+     * with CLEAR ops.
      */
     async clearBucketLeading(lastNotPut, boundaryDocId, bucketContext, collection, context) {
         let opCountDiff = 0;
+        let before = emptyBucketStats();
+        let after = emptyBucketStats();
         const session = this.db.client.startSession();
         try {
             let done = false;
@@ -411,26 +807,36 @@ export class MongoCompactorV3 extends MongoCompactor {
                 const batch = await this.clearLeadingFullDocuments(session, lastNotPut, boundaryDocId, bucketContext, collection, context);
                 done = batch.done;
                 opCountDiff += batch.opCountDiff;
+                before = combineAdjacentStats(before, batch.before);
+                after = combineAdjacentStats(after, batch.after);
             }
             // The final step is to process the "boundary" document: It may contain some CLEAR/MOVE/REMOVE operations,
             // potentially followed by PUT operations. This is only a single document, so no need for batching.
-            opCountDiff += await this.clearBoundaryDocument(session, lastNotPut, boundaryDocId, bucketContext, collection, context);
+            const boundaryResult = await this.clearBoundaryDocument(session, lastNotPut, boundaryDocId, bucketContext, collection, context);
+            opCountDiff += boundaryResult.opCountDiff;
+            before = combineAdjacentStats(before, boundaryResult.before);
+            after = combineAdjacentStats(after, boundaryResult.after);
         }
         finally {
             await session.endSession();
         }
-        return opCountDiff;
+        return { opCountDiff, before, after };
     }
     async clearLeadingFullDocuments(session, lastNotPut, boundaryDocId, bucketContext, collection, context) {
         const bucket = bucketContext.key.bucket;
         this.signal?.throwIfAborted();
-        const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut]);
+        let prepared;
         let done = false;
         let opCountDiff = 0;
+        let before = emptyBucketStats();
+        let after = emptyBucketStats();
         await session.withTransaction(async () => {
             done = false;
             opCountDiff = 0;
+            before = emptyBucketStats();
+            after = emptyBucketStats();
             const oldStoragePaths = [];
+            let oldStorageBytes = 0n;
             const query = collection.find({
                 _id: {
                     $gte: bucketContext.minId,
@@ -444,6 +850,7 @@ export class MongoCompactorV3 extends MongoCompactor {
                     min_op: 1,
                     checksum: 1,
                     count: 1,
+                    size: 1,
                     target_op: 1,
                     has_clear_op: 1,
                     storage_ref: 1
@@ -456,14 +863,21 @@ export class MongoCompactorV3 extends MongoCompactor {
             let lastDocId = null;
             let clearOpCount = 0;
             let gotNonClearOp = false;
+            const inputStats = emptyBucketStats();
             for await (const doc of query.stream()) {
                 if (doc.min_op > lastNotPut) {
                     throw new ReplicationAssertionError(`Unexpected document before CLEAR boundary with min_op ${doc.min_op} > ${lastNotPut} in bucket ${bucket}`);
                 }
                 lastDocId = doc._id;
+                const documentStats = statsForDocument(doc);
+                inputStats.count += documentStats.count;
+                inputStats.bytes += documentStats.bytes;
+                inputStats.chunks += documentStats.chunks;
+                inputStats.checksum = addChecksums(inputStats.checksum, documentStats.checksum);
                 if (doc.storage_ref) {
                     oldStoragePaths.push(doc.storage_ref.path);
                 }
+                oldStorageBytes += ObjectStorageUsage.bytes(doc);
                 // The compaction scan established that every operation before the
                 // boundary is MOVE/REMOVE/CLEAR. Root metadata is sufficient to fold
                 // whole documents into one CLEAR, so avoid downloading their payloads.
@@ -486,6 +900,7 @@ export class MongoCompactorV3 extends MongoCompactor {
                 done = true;
                 return;
             }
+            prepared ??= await this.prepareCompactionUploads(bucket, context, [lastNotPut]);
             this.logger.info(`Flushing CLEAR for ${clearedOpCount} ops at ${lastDocId?.o}`);
             await collection.deleteMany({
                 _id: {
@@ -505,21 +920,29 @@ export class MongoCompactorV3 extends MongoCompactor {
             });
             await collection.insertOne(persisted.documents[0], { session });
             await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session);
+            await this.recordObjectStorageReplacement(oldStorageBytes, persisted.documents, context.definitionId, session);
             opCountDiff = -clearedOpCount + 1;
+            before = inputStats;
+            after = statsForDocuments(persisted.documents);
         }, {
             writeConcern: { w: 'majority' },
             readConcern: { level: 'snapshot' }
         });
-        return { done, opCountDiff };
+        return { done, opCountDiff, before, after };
     }
     async clearBoundaryDocument(session, lastNotPut, boundaryDocId, bucketContext, collection, context) {
         const bucket = bucketContext.key.bucket;
         this.signal?.throwIfAborted();
         const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut, boundaryDocId.o]);
         let opCountDiff = 0;
+        let before = emptyBucketStats();
+        let after = emptyBucketStats();
         await session.withTransaction(async () => {
             opCountDiff = 0;
+            before = emptyBucketStats();
+            after = emptyBucketStats();
             const oldStoragePaths = [];
+            let oldStorageBytes = 0n;
             const query = collection.find({
                 // This is a range query, but should only ever return two documents:
                 // 1. The CLEAR op from the previous clearLeadingFullDocuments.
@@ -536,6 +959,7 @@ export class MongoCompactorV3 extends MongoCompactor {
                     min_op: 1,
                     checksum: 1,
                     count: 1,
+                    size: 1,
                     target_op: 1,
                     ops: 1,
                     storage_ref: 1
@@ -547,15 +971,22 @@ export class MongoCompactorV3 extends MongoCompactor {
             let clearedOpCount = 0;
             let maxTargetOp = null;
             const boundarySurvivors = [];
+            const inputStats = emptyBucketStats();
             for await (const doc of query.stream()) {
                 docsRead++;
                 if (docsRead > 2) {
                     throw new ReplicationAssertionError(`Unexpected extra document before CLEAR boundary in bucket ${bucket}`);
                 }
+                const documentStats = statsForDocument(doc);
+                inputStats.count += documentStats.count;
+                inputStats.bytes += documentStats.bytes;
+                inputStats.chunks += documentStats.chunks;
+                inputStats.checksum = addChecksums(inputStats.checksum, documentStats.checksum);
                 const isBoundaryDoc = doc._id.o == boundaryDocId.o;
                 if (doc.storage_ref) {
                     oldStoragePaths.push(doc.storage_ref.path);
                 }
+                oldStorageBytes += ObjectStorageUsage.bytes(doc);
                 await hydrateBucketDataDocuments([doc], this.storage.objectStorage, { signal: this.signal });
                 maxTargetOp = maxOpId(maxTargetOp, doc.target_op);
                 for (const op of loadBucketDataDocument(context, doc)) {
@@ -605,12 +1036,15 @@ export class MongoCompactorV3 extends MongoCompactor {
             });
             await collection.insertMany(persisted.documents, { session });
             await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session);
+            await this.recordObjectStorageReplacement(oldStorageBytes, persisted.documents, context.definitionId, session);
             opCountDiff = -clearedOpCount + 1;
+            before = inputStats;
+            after = statsForDocuments(persisted.documents);
         }, {
             writeConcern: { w: 'majority' },
             readConcern: { level: 'snapshot' }
         });
-        return opCountDiff;
+        return { opCountDiff, before, after };
     }
     /**
      * Reserve stable object paths before starting a retryable MongoDB transaction.
@@ -637,6 +1071,16 @@ export class MongoCompactorV3 extends MongoCompactor {
         await this.objectStorageLifecycle.publishUploads(uploads, session);
         await this.objectStorageLifecycle.retire(Array.from(oldStoragePaths).filter((path) => !newStoragePaths.has(path)), session);
     }
+    async recordObjectStorageReplacement(oldBytes, newDocuments, definitionId, session) {
+        if (!this.storage.objectStorage) {
+            return;
+        }
+        let newBytes = 0n;
+        for (const document of newDocuments) {
+            newBytes += ObjectStorageUsage.bytes(document);
+        }
+        await this.objectStorageUsage.applyDelta(definitionId, newBytes - oldBytes, session);
+    }
     async persistBucketData(bucket, chunks, context, preparedUploads, options) {
         const serializedChunks = chunks.map((chunk) => serializeBucketData(bucket, chunk, options));
         if (!this.storage.objectStorage) {
@@ -679,7 +1123,7 @@ export class MongoCompactorV3 extends MongoCompactor {
                 return serialized;
             }
             const { ops, ...metadata } = serialized;
-            const { fileSize } = await store.store(upload.path, ops);
+            const { fileSize } = await store.store(upload.path, ops, { signal: this.signal });
             storagePaths.add(upload.path);
             return {
                 ...metadata,