@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts

@@ -1,5 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
+import { BucketDefinitionId, EventDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
 import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
 import { CommonSourceTableDocument } from '../models.js';
 import {
@@ -7,58 +7,134 @@ import {
   BucketParameterDocumentV3,
   BucketStateDocumentV3,
   CurrentDataDocumentV3,
+  CustomCheckpointRequestDocumentV3,
+  ObjectStorageDeletionMarker,
+  ObjectStorageUsageDocument,
   SourceTableDocumentV3,
   SyncConfigDefinition
 } from './models.js';
 
+export const OBJECT_STORAGE_USAGE_COLLECTION = 'object_storage_usage';
+export type CustomCheckpointRequestCollectionSpecifier = {
+  replicationStreamId: number;
+  eventId: EventDefinitionId;
+};
+
 export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
+  constructor(
+    upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0],
+    storageConfig: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[1]
+  ) {
+    super(upstream, storageConfig);
+  }
+
   get syncConfigDefinitions() {
     return this.db.collection<SyncConfigDefinition>('sync_config');
   }
 
-  sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
+  sourceRecords(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
     const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
     return this.db.collection<CurrentDataDocumentV3>(collectionName);
   }
 
-  async listSourceRecordCollectionsV3(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
+  async listSourceRecordCollections(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
     return this.listCollectionsByPrefix<CurrentDataDocumentV3>(`source_records_${replicationStreamId}_`);
   }
 
-  async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
-    await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex(
+  async initializeSourceRecordsCollection(
+    replicationStreamId: number,
+    sourceTableId: mongo.ObjectId,
+    session?: mongo.ClientSession
+  ) {
+    await this.sourceRecords(replicationStreamId, sourceTableId).createIndex(
       {
         pending_delete: 1
       },
       {
         partialFilterExpression: { pending_delete: { $exists: true } },
-        name: 'pending_delete'
+        name: 'pending_delete',
+        session
       }
     );
   }
 
   commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument> {
-    return this.sourceTablesV3(replicationStreamId) as mongo.Collection<CommonSourceTableDocument>;
+    return this.sourceTables(replicationStreamId) as any as mongo.Collection<CommonSourceTableDocument>;
   }
 
-  bucketStateV3(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
+  bucketState(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
     return this.db.collection(`bucket_state_${replicationStreamId}`);
   }
 
-  parameterIndexV3(
-    replicationStreamId: number,
-    indexId: ParameterIndexId
-  ): mongo.Collection<BucketParameterDocumentV3> {
+  parameterIndex(replicationStreamId: number, indexId: ParameterIndexId): mongo.Collection<BucketParameterDocumentV3> {
     return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`);
   }
 
-  sourceTablesV3(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
+  sourceTables(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
     return this.db.collection<SourceTableDocumentV3>(this.sourceTableCollectionName(replicationStreamId));
   }
 
+  bucketData(replicationStreamId: number, definitionId: BucketDefinitionId): mongo.Collection<BucketDataDocumentV3> {
+    return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
+  }
+
+  pendingObjectStorageDeletes(replicationStreamId: number): mongo.Collection<ObjectStorageDeletionMarker> {
+    return this.db.collection<ObjectStorageDeletionMarker>(`pending_object_storage_deletes_${replicationStreamId}`);
+  }
+
+  get objectStorageUsage(): mongo.Collection<ObjectStorageUsageDocument> {
+    return this.db.collection<ObjectStorageUsageDocument>(OBJECT_STORAGE_USAGE_COLLECTION);
+  }
+
+  customCheckpointRequests(options: CustomCheckpointRequestCollectionSpecifier) {
+    const collectionName = this.customCheckpointRequestCollectionName(options);
+    return this.db.collection<CustomCheckpointRequestDocumentV3>(collectionName);
+  }
+
+  async initializeCustomCheckpointRequestsCollection(options: CustomCheckpointRequestCollectionSpecifier) {
+    // createIndexes also creates the collection on first use. Callers invoke
+    // this lazily before the transaction which writes the first checkpoint.
+    await this.customCheckpointRequests(options).createIndexes([
+      {
+        key: { user_id: 1 },
+        name: 'user_unique',
+        unique: true
+      },
+      {
+        // Find checkpoint changes over an operation-id range.
+        key: { op_id: 1 },
+        name: 'op_id'
+      },
+      {
+        // Limit retention scans to temporary client-requested checkpoints.
+        key: { checkpoint_requested_at: 1 },
+        name: 'checkpoint_requested_at',
+        partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+      }
+    ]);
+  }
+
+  listCustomCheckpointRequestCollections(replicationStreamId: number) {
+    return this.upstream.listCustomCheckpointRequestCollections(replicationStreamId);
+  }
+
+  listBucketDataCollections(replicationStreamId: number) {
+    return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
+  }
+
+  async listParameterIndexCollections(replicationStreamId: number) {
+    const prefix = `parameter_index_${replicationStreamId}_`;
+    const collections = await this.listCollectionsByPrefix(prefix);
+
+    return collections.map((collection) => ({
+      collection,
+      indexId: collection.collectionName.slice(prefix.length)
+    }));
+  }
+
   async initializeStreamStorage(replicationStreamId: number) {
-    const sourceTables = this.sourceTablesV3(replicationStreamId);
-    const bucketState = this.bucketStateV3(replicationStreamId);
+    const sourceTables = this.sourceTables(replicationStreamId);
+    const bucketState = this.bucketState(replicationStreamId);
     await sourceTables.createIndex(
       {
         connection_id: 1,
@@ -85,33 +161,18 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
       },
       { name: 'bucket_updates', unique: true }
     );
+    // V3 workers claim only buckets that have reached their scheduled check
+    // time, keeping scheduled scans proportional to pending work.
     await bucketState.createIndex(
+      { next_compact_check: 1 },
       {
-        'estimate_since_compact.count': -1
-      },
-      { name: 'dirty_count' }
+        name: 'next_compact_check',
+        partialFilterExpression: { next_compact_check: { $exists: true } }
+      }
     );
   }
 
-  bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId) {
-    return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
-  }
-
-  listBucketDataCollectionsV3(replicationStreamId: number) {
-    return this.upstream.listBucketDataCollectionsV3(replicationStreamId);
-  }
-
-  async listParameterIndexCollectionsV3(
-    replicationStreamId: number
-  ): Promise<{ collection: mongo.Collection<BucketParameterDocumentV3>; indexId: ParameterIndexId }[]> {
-    const prefix = `parameter_index_${replicationStreamId}_`;
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
-
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => ({
-        collection: this.db.collection<BucketParameterDocumentV3>(collection.name),
-        indexId: collection.name.slice(prefix.length)
-      }));
+  customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
+    return this.upstream.customCheckpointRequestCollectionName(options);
   }
 }

package/src/storage/implementation/v3/bucket-format.ts

@@ -0,0 +1,99 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { bson, InternalOpId } from '@powersync/service-core';
+import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
+import { BucketDataDocumentV3, BucketOperation } from './models.js';
+
+/**
+ * Serialize a non-empty list of operations in strictly ascending `o` order.
+ * Preserving that order establishes the {@link BucketDataDocumentV3} range
+ * invariants used by metadata-only queries.
+ */
+export function serializeBucketData(
+  bucket: string,
+  operations: BucketDataDoc[],
+  options?: { targetOp?: InternalOpId | null }
+): BucketDataDocumentV3 {
+  const minOp = operations[0].o;
+  const maxOp = operations[operations.length - 1].o;
+
+  let totalChecksum = 0n;
+  let maxTargetOp: InternalOpId | null = options?.targetOp ?? null;
+  let hasClearOp = false;
+
+  const ops: BucketOperation[] = operations.map((op) => {
+    totalChecksum += op.checksum;
+
+    if (op.op == 'CLEAR') {
+      hasClearOp = true;
+    }
+
+    return {
+      o: op.o,
+      op: op.op,
+      source_table: op.source_table,
+      source_key: op.source_key,
+      subkey: op.subkey,
+      table: op.table,
+      row_id: op.row_id,
+      checksum: op.checksum,
+      data: op.data
+    };
+  });
+
+  const size = bson.calculateObjectSize(ops);
+
+  return {
+    _id: {
+      b: bucket,
+      o: maxOp
+    },
+    min_op: minOp,
+    checksum: totalChecksum,
+    count: operations.length,
+    size,
+    target_op: maxTargetOp,
+    ...(hasClearOp ? { has_clear_op: true as const } : {}),
+    ops
+  };
+}
+
+export function* loadBucketDataDocument(
+  context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>,
+  doc: BucketDataDocumentV3
+): Generator<BucketDataDoc> {
+  const { _id, ops } = doc;
+  if (!ops) {
+    throw new ServiceAssertionError(
+      `Missing ops array on BucketDataDocumentV3 at _id.o=${_id.o}. Callers must patch doc.ops from S3 before calling this function.`
+    );
+  }
+  const bucketKey = {
+    ...context,
+    bucket: _id.b
+  };
+
+  for (const op of ops) {
+    yield {
+      bucketKey,
+      o: op.o,
+      op: op.op,
+      source_table: op.source_table,
+      source_key: op.source_key,
+      subkey: op.subkey,
+      table: op.table,
+      row_id: op.row_id,
+      checksum: op.checksum,
+      data: op.data
+    };
+  }
+}
+
+export function maxOpId(a: InternalOpId | null | undefined, b: InternalOpId | null | undefined): InternalOpId | null {
+  if (a == null) {
+    return b ?? null;
+  }
+  if (b == null) {
+    return a ?? null;
+  }
+  return a > b ? a : b;
+}

package/src/storage/implementation/v3/chunking.ts

@@ -0,0 +1,40 @@
+import { bson } from '@powersync/service-core';
+import { BucketDataDoc } from '../common/BucketDataDoc.js';
+
+export const DEFAULT_MAX_DOC_SIZE_BYTES = 1024 * 1024; // 1MB
+
+/**
+ * Split an array of bucket data operations into chunks where each chunk's
+ * total data size stays close to `maxDocSizeBytes`.
+ *
+ * The threshold is a target, not a hard ceiling — a single operation whose
+ * `data` field exceeds the limit is placed in its own chunk and is NOT split.
+ */
+export function chunkBucketData(
+  operations: BucketDataDoc[],
+  maxDocSizeBytes = DEFAULT_MAX_DOC_SIZE_BYTES
+): BucketDataDoc[][] {
+  const chunks: BucketDataDoc[][] = [];
+  let currentChunk: BucketDataDoc[] = [];
+  let currentSize = 0;
+
+  for (const op of operations) {
+    // Add 5 to account for array index up to 4 digits (9999) plus null byte.
+    const opSize = bson.calculateObjectSize(op) + 5;
+
+    if (currentSize + opSize > maxDocSizeBytes && currentChunk.length > 0) {
+      chunks.push(currentChunk);
+      currentChunk = [];
+      currentSize = 0;
+    }
+
+    currentChunk.push(op);
+    currentSize += opSize;
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+
+  return chunks;
+}

package/src/storage/implementation/v3/compact-utils.ts

@@ -0,0 +1,320 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { addChecksums, InternalOpId } from '@powersync/service-core';
+import { BucketDataDoc } from '../common/BucketDataDoc.js';
+import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js';
+import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js';
+
+/**
+ * Perform chunk compaction if at least this many chunks have been added since
+ * the latest compaction.
+ *
+ * A lower value increases compaction frequency and can increase chunk rewrites
+ * and object-storage operations. A higher value leaves more small chunks for
+ * longer, which can hurt sync performance.
+ */
+const MERGE_CHUNKS_THRESHOLD = 8;
+
+export const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000;
+
+export interface PendingCompactionGroup {
+  /**
+   * Input documents are ordered from oldest to newest, matching `ops`.
+   * Keeping the inputs intact lets unchanged singletons retain their object.
+   */
+  inputs: BucketDataDocumentV3[];
+  ops: BucketDataDoc[];
+  changed: boolean;
+  targetOp: InternalOpId | null;
+}
+
+export interface CompactIntervalConfig {
+  readonly minCompactChunkIntervalMs: number;
+  readonly minCompactFullIntervalMs: number;
+  readonly maxCompactFullIntervalMs: number;
+}
+export interface CompactTargetConfig {
+  readonly maxOpIdCap: InternalOpId | undefined;
+}
+
+export enum CompactionKind {
+  Full = 'full',
+  Chunks = 'chunks'
+}
+
+export interface CompactionDecision {
+  kind: CompactionKind | null;
+  nextCompactCheck: mongo.Document;
+}
+
+export interface ScheduledCompactionOptions {
+  /** Process checks scheduled this far after the captured job start. */
+  dueAheadMs?: number;
+  /** Used by initial replication, which must not run a full compact. */
+  forceKind?: CompactionKind;
+}
+
+export class CompactionContext {
+  constructor(
+    readonly lease: CompactionLease,
+    readonly kind: CompactionKind,
+    readonly decision: CompactionDecision,
+    readonly rescheduleNotBefore: Date | undefined,
+    readonly targetOp: InternalOpId
+  ) {}
+
+  get state() {
+    return this.lease.state;
+  }
+
+  get startedAt() {
+    return this.lease.startedAt;
+  }
+
+  get lastOp() {
+    return this.lease.lastOp;
+  }
+}
+
+export interface BucketStats {
+  count: number;
+  bytes: bigint;
+  chunks: number;
+}
+
+/** Bucket stats read from bucket-data documents, including their checksum. */
+export interface BucketStatsWithChecksum extends BucketStats {
+  checksum: number;
+}
+
+export interface CompactionResult {
+  /** Metadata cached at compacted_state.op_id. */
+  compactedState: BucketStatsWithChecksum;
+  /** Complete bucket metadata through the op head captured at claim time. */
+  bucketStats: BucketStats;
+}
+
+export function bucketStats(state: BucketStateDocumentV3): BucketStats {
+  return {
+    count: state.bucket_stats.count,
+    bytes: state.bucket_stats.bytes,
+    chunks: state.bucket_stats.chunks
+  };
+}
+
+export function emptyBucketStats(): BucketStatsWithChecksum {
+  return { count: 0, bytes: 0n, chunks: 0, checksum: 0 };
+}
+
+export function statsForDocument(
+  document: Pick<BucketDataDocumentV3, 'count' | 'size' | 'checksum'>
+): BucketStatsWithChecksum {
+  return {
+    count: document.count,
+    bytes: BigInt(document.size),
+    chunks: 1,
+    checksum: addChecksums(0, Number(document.checksum))
+  };
+}
+
+export function statsForDocuments(
+  documents: Iterable<Pick<BucketDataDocumentV3, 'count' | 'size' | 'checksum'>>
+): BucketStatsWithChecksum {
+  let result = emptyBucketStats();
+  for (const document of documents) {
+    result = combineAdjacentStats(result, statsForDocument(document));
+  }
+  return result;
+}
+
+/** A scheduled bucket always has writes awaiting a full compact. */
+export function firstUncompactedWrite(state: BucketStateDocumentV3): Date {
+  if (state.first_uncompacted_write == null) {
+    throw new ReplicationAssertionError(`Scheduled V3 bucket ${state._id.b} has no first uncompacted write`);
+  }
+  return state.first_uncompacted_write;
+}
+
+export function combineChunkStats(
+  previous: NonNullable<BucketStateDocumentV3['compacted_state']>,
+  compactedTail: BucketStatsWithChecksum,
+  overlappingCompactedChunk: BucketStatsWithChecksum
+): BucketStatsWithChecksum {
+  return {
+    count: previous.count - overlappingCompactedChunk.count + compactedTail.count,
+    bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes,
+    chunks: previous.chunks - 1 + compactedTail.chunks,
+    checksum: addChecksums(
+      addChecksums(Number(previous.checksum), -overlappingCompactedChunk.checksum),
+      compactedTail.checksum
+    )
+  };
+}
+
+export function combineAdjacentStats(
+  first: BucketStatsWithChecksum,
+  second: BucketStatsWithChecksum
+): BucketStatsWithChecksum {
+  return {
+    count: first.count + second.count,
+    bytes: first.bytes + second.bytes,
+    chunks: first.chunks + second.chunks,
+    checksum: addChecksums(first.checksum, second.checksum)
+  };
+}
+
+export function applyCompactionDelta(
+  total: BucketStats,
+  before: BucketStatsWithChecksum,
+  after: BucketStatsWithChecksum
+): BucketStats {
+  return {
+    count: total.count - before.count + after.count,
+    bytes: total.bytes - before.bytes + after.bytes,
+    chunks: total.chunks - before.chunks + after.chunks
+  };
+}
+
+/** Replace one stored range in an accumulated statistics snapshot. */
+export function applyStatsReplacement(
+  total: BucketStatsWithChecksum,
+  before: BucketStatsWithChecksum,
+  after: BucketStatsWithChecksum
+): BucketStatsWithChecksum {
+  return {
+    ...applyCompactionDelta(total, before, after),
+    checksum: addChecksums(addChecksums(total.checksum, -before.checksum), after.checksum)
+  };
+}
+
+/**
+ * This checks that the bucket state hasn't changed by a concurrent write since we checked.
+ *
+ * If it has changed, we'll re-check in the next batch.
+ */
+export function unclaimedSnapshotFilter(state: BucketStateDocumentV3): mongo.Filter<BucketStateDocumentV3> {
+  return {
+    _id: state._id,
+    last_op: state.last_op,
+    next_compact_check: state.next_compact_check,
+    first_uncompacted_write: state.first_uncompacted_write ?? { $exists: false },
+    bucket_stats: state.bucket_stats,
+    compacted_state: state.compacted_state ?? { $exists: false },
+    last_full_compact: state.last_full_compact ?? { $exists: false },
+    ...AVAILABLE_LEASE_EXPR
+  };
+}
+
+/**
+ * Calculate the earliest full compaction time from the first uncompacted
+ * write, bounded by the maximum retention interval.
+ */
+export function fullCompactionCheckAt(state: BucketStateDocumentV3, config: CompactIntervalConfig): Date {
+  const firstWrite = firstUncompactedWrite(state);
+  const stats = bucketStats(state);
+  const lastFull = state.last_full_compact;
+
+  // The number of operations since the last full compact.
+  // We may make this more specific in the future, to track new updates and deletes only, ignoring
+  // full new inserts, but that requires more granular tracking when replicating.
+  const uncompactedCount = lastFull == null ? stats.count : Math.max(0, stats.count - lastFull.count);
+  const compactedRows = lastFull?.puts ?? 0;
+
+  // If no full compact has ever been performed: ratio = 1, compact after minCompactFullIntervalMs.
+  // If every row has been updated or deleted exactly once since the last full compact: ratio = 0.5, compact after minCompactFullIntervalMs * 2.
+  // If 10% of rows has been updated since the last full compact, compact after minCompactFullIntervalMs * 11.
+  // If every row has been updated multiple times, the ratio tends closer to 1 again.
+  const ratio = uncompactedCount == 0 ? 0 : uncompactedCount / (compactedRows + uncompactedCount);
+  const fullIntervalMs = ratio == 0 ? config.maxCompactFullIntervalMs : config.minCompactFullIntervalMs / ratio;
+  return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, config.maxCompactFullIntervalMs));
+}
+
+export function chooseCompactionKind(
+  state: BucketStateDocumentV3,
+  now: Date,
+  config: CompactIntervalConfig
+): CompactionDecision {
+  // For chunk compaction, we consider the number of chunks added.
+  // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added.
+  // A future policy could also use bytes per chunk or records per chunk to
+  // decide when to compact.
+  const fullCheckAt = fullCompactionCheckAt(state, config);
+  // Schedule a little late so a worker using a slightly earlier clock does
+  // not wake before the exact full-compaction condition is true.
+  const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS);
+  const compacted = state.compacted_state;
+  const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (compacted?.chunks ?? 0));
+  const shouldCompactChunks = chunksSinceCompact >= MERGE_CHUNKS_THRESHOLD;
+  const canCheckChunks =
+    compacted == null || now.getTime() - compacted.at.getTime() >= config.minCompactChunkIntervalMs;
+  // Too few new chunks cannot make chunk compaction eligible. Do not poll this
+  // bucket at the chunk-compaction interval; only wake it for its full-compact check.
+  const nextCompactCheck: mongo.Document = !shouldCompactChunks
+    ? fullCheckWithMargin
+    : {
+        $min: [
+          fullCheckWithMargin,
+          { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: config.minCompactChunkIntervalMs } }
+        ]
+      };
+  let kind: CompactionKind | null = null;
+  if (now >= fullCheckAt) {
+    kind = CompactionKind.Full;
+  } else if (canCheckChunks && shouldCompactChunks) {
+    kind = CompactionKind.Chunks;
+  }
+  return {
+    kind,
+    nextCompactCheck
+  };
+}
+
+export function forcedCompactionKind(
+  state: BucketStateDocumentV3,
+  forceKind: CompactionKind | undefined,
+  config: CompactTargetConfig
+): CompactionKind | null {
+  if (forceKind == null) {
+    return null;
+  }
+  const maxOpId = config.maxOpIdCap == null || state.last_op < config.maxOpIdCap ? state.last_op : config.maxOpIdCap;
+  return state.compacted_state == null || state.compacted_state.op_id < maxOpId ? forceKind : null;
+}
+
+/**
+ * Read one bounded prefix from a compaction cursor.
+ *
+ * The document that would cross the byte limit is deliberately not returned:
+ * pagination resumes past the last returned `_id`, so that document remains
+ * eligible for the next query. The first document is always accepted to
+ * ensure progress when a single document exceeds the configured byte limit.
+ *
+ * `hasMore` is conservative when the document limit is reached. An extra empty
+ * query is preferable to exhausting the cursor just to determine whether the
+ * limited MongoDB query contained another document.
+ */
+export async function readCompactionBatch(
+  cursor: mongo.AggregationCursor<BucketDataDocumentV3>,
+  options: { byteLimit: number; documentLimit: number }
+): Promise<{ documents: BucketDataDocumentV3[]; hasMore: boolean }> {
+  const documents: BucketDataDocumentV3[] = [];
+  let cumulativeBytes = 0;
+
+  try {
+    for await (const document of cursor) {
+      if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) {
+        return { documents, hasMore: true };
+      }
+
+      documents.push(document);
+      cumulativeBytes += document.size;
+
+      if (documents.length >= options.documentLimit) {
+        return { documents, hasMore: true };
+      }
+    }
+    return { documents, hasMore: false };
+  } finally {
+    await cursor.close();
+  }
+}

package/src/storage/implementation/v3/compaction-constants.ts

@@ -0,0 +1,5 @@
+/**
+ * Writers use this delay before scheduling their first chunk compaction.
+ * Initial replication includes the same delay when it picks up that work.
+ */
+export const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS = 5 * 60 * 1000;