@powersync/service-module-mongodb-storage 0.20.0 → 0.21.0

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -2,6 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import {
   BaseObserver,
+  logger as defaultLogger,
   DO_NOT_LOG,
   Logger,
   ReplicationAbortedError,
@@ -11,14 +12,15 @@ import {
   BroadcastIterable,
   CHECKPOINT_INVALIDATE_ALL,
   CheckpointChanges,
+  CompactInitialReplicationOptions,
+  CompactInitialReplicationResults,
   GetCheckpointChangesOptions,
   InternalOpId,
   mergeAsyncIterables,
-  PopulateChecksumCacheOptions,
-  PopulateChecksumCacheResults,
   ReplicationCheckpoint,
   ReplicationStreamStorageIds,
   storage,
+  SyncRuleState,
   utils,
   WatchWriteCheckpointOptions
 } from '@powersync/service-core';
@@ -28,6 +30,10 @@ import { LRUCache } from 'lru-cache';
 import * as timers from 'timers/promises';
 import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
+import {
+  MongoGetCheckpointChangesOptions,
+  MongoSyncBucketStorageCheckpoint
+} from './common/MongoSyncBucketStorageCheckpoint.js';
 import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js';
 import type { VersionedPowerSyncMongo } from './db.js';
 import { StorageConfig } from './models.js';
@@ -37,7 +43,7 @@ import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
 import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
 import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
-import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
+import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
 
 export interface MongoSyncBucketStorageOptions {
@@ -50,6 +56,16 @@ export interface MongoSyncBucketStorageOptions {
   inlineThresholdBytes?: number;
 }
 
+/**
+ * The stream state read for a checkpoint. All fields must come from the same snapshot.
+ */
+export interface MongoCheckpointState {
+  checkpoint: InternalOpId;
+  lsn: string | null;
+  /** See {@link MongoSyncBucketStorageCheckpoint.parameterChangesInvalidBefore}. */
+  parameterChangesInvalidBefore: InternalOpId;
+}
+
 interface InternalCheckpointChanges extends CheckpointChanges {
   updatedWriteCheckpoints: Map<string, bigint>;
   invalidateWriteCheckpoints: boolean;
@@ -112,12 +128,27 @@ export abstract class MongoSyncBucketStorage
     this.clearBatchThrottleRate = options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
     this.db = factory.db.versioned(this.storageConfig);
     this.checksums = this.createMongoChecksums(options);
-    this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
+    this.writeCheckpointAPI = this.createWriteCheckpointAPI({
       db: this.db,
-      mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
-      sync_rules_id: replicationStreamId
+      writeCheckpointMode: { mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED },
+      replicationStreamId
     });
-    this.logger = replicationStream.logger;
+
+    // Include both replication stream name and sync config version labels in log prefix.
+    // Examples:
+    // No sync config with a label: [stream]
+    // All sync configs have a label: [stream][v5,v6]
+    // Mixed: [stream][?,v6]
+    const hasVersionLabel = replicationStream.syncConfigContent.some((config) => config.version_label != null);
+    const versionLabels = hasVersionLabel
+      ? replicationStream.syncConfigContent.map((config) => config.version_label ?? '?')
+      : [];
+    this.logger =
+      versionLabels.length == 0
+        ? replicationStream.logger
+        : defaultLogger.child({
+            prefix: `[${replicationStreamName}][${versionLabels.join(',')}] `
+          });
   }
 
   /**
@@ -128,11 +159,20 @@ export abstract class MongoSyncBucketStorage
   abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
 
   protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
+  protected createWriteCheckpointAPI(options: MongoCheckpointAPIOptions): MongoWriteCheckpointAPI {
+    return new MongoWriteCheckpointAPI(options);
+  }
+
   protected abstract createMongoParameterCompactor(
     checkpoint: InternalOpId,
     options: storage.CompactOptions
   ): MongoParameterCompactor;
 
+  /** MongoDB parameter compaction uses a persisted operation-id cursor. */
+  public supportsIncrementalParameterCompaction(): boolean {
+    return true;
+  }
+
   get writeCheckpointMode() {
     return this.writeCheckpointAPI.writeCheckpointMode;
   }
@@ -144,8 +184,8 @@ export abstract class MongoSyncBucketStorage
     return this.replicationStream.storageIds;
   }
 
-  setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
-    this.writeCheckpointAPI.setWriteCheckpointMode(mode);
+  setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    this.writeCheckpointAPI.setWriteCheckpointMode(config);
   }
 
   createManagedWriteCheckpoints(
@@ -178,9 +218,7 @@ export abstract class MongoSyncBucketStorage
     return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint();
   }
 
-  protected abstract fetchCheckpointState(
-    session: mongo.ClientSession
-  ): Promise<{ checkpoint: bigint; lsn: string | null } | null>;
+  protected abstract fetchCheckpointState(session: mongo.ClientSession): Promise<MongoCheckpointState | null>;
 
   async getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null> {
     return await this.db.client.withSession({ snapshot: true }, async (session) => {
@@ -197,7 +235,14 @@ export abstract class MongoSyncBucketStorage
       if (clusterTime == null) {
         throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
       }
-      return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime, clusterTime);
+      return new MongoReplicationCheckpoint(
+        this,
+        state.checkpoint,
+        state.lsn,
+        snapshotTime,
+        clusterTime,
+        state.parameterChangesInvalidBefore
+      );
     });
   }
 
@@ -235,6 +280,7 @@ export abstract class MongoSyncBucketStorage
       markRecordUnavailable: options.markRecordUnavailable,
       hooks: options.hooks,
       tracer: options.tracer,
+      signal: options.signal,
       objectStorage: this.objectStorage,
       inlineThresholdBytes: this.inlineThresholdBytes
     };
@@ -329,6 +375,7 @@ export abstract class MongoSyncBucketStorage
   protected abstract clearBucketState(signal?: AbortSignal): Promise<void>;
 
   protected abstract clearSourceTables(signal?: AbortSignal): Promise<void>;
+  protected async clearCustomCheckpointRequests(_signal?: AbortSignal): Promise<void> {}
   protected abstract clearSyncRuleState(): Promise<void>;
 
   async clear(options?: storage.ClearStorageOptions): Promise<void> {
@@ -345,6 +392,7 @@ export abstract class MongoSyncBucketStorage
     await this.clearSourceRecords(signal);
     await this.clearBucketState(signal);
     await this.clearSourceTables(signal);
+    await this.clearCustomCheckpointRequests(signal);
 
     this.#storageInitialized = false;
   }
@@ -367,43 +415,30 @@ export abstract class MongoSyncBucketStorage
   async compact(options?: storage.CompactOptions) {
     let maxOpId = options?.maxOpId;
     if (maxOpId == null) {
+      // For PROCESSING streams, this will be undefined.
       const checkpoint = await this.getCheckpointInternal();
       maxOpId = checkpoint?.checkpoint ?? undefined;
     }
     await this.createMongoCompactor({ ...options, maxOpId, logger: this.logger }).compact();
 
-    if (maxOpId != null && options?.compactParameterData) {
-      await this.createMongoParameterCompactor(maxOpId, options).compact();
+    if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.ACTIVE) {
+      // Use the stream-scoped logger, matching bucket compaction above.
+      await this.createMongoParameterCompactor(maxOpId, { ...options, logger: this.logger }).compact();
     }
   }
 
+  abstract compactInitialReplication(
+    options: CompactInitialReplicationOptions
+  ): Promise<CompactInitialReplicationResults>;
+
   /**
    * The highest op id persisted for this stream, whether or not covered by a checkpoint.
    *
-   * Used as the default `maxOpId` for {@link populatePersistentChecksumCache}, which runs after
+   * Used as the default `maxOpId` for {@link compactInitialReplication}, which runs after
    * initial replication but before the first checkpoint exists.
    */
   protected abstract fetchPersistedOpHead(): Promise<InternalOpId | null>;
 
-  async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
-    this.logger.info(`Populating persistent checksum cache...`);
-    const start = Date.now();
-    const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
-    const compactor = this.createMongoCompactor({
-      ...options,
-      maxOpId,
-      memoryLimitMB: 0,
-      logger: this.logger
-    });
-
-    const result = await compactor.populateChecksums({
-      minBucketChanges: options.minBucketChanges ?? 10
-    });
-    const duration = Date.now() - start;
-    this.logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`);
-    return result;
-  }
-
   private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ReplicationCheckpoint> {
     if (signal.aborted) {
       return;
@@ -580,13 +615,31 @@ export abstract class MongoSyncBucketStorage
   }
 
   protected abstract getParameterBucketChangesImpl(
-    options: GetCheckpointChangesOptions
+    options: MongoGetCheckpointChangesOptions
   ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
 
   private async getParameterBucketChanges(
     options: GetCheckpointChangesOptions
   ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
-    return this.getParameterBucketChangesImpl(options);
+    const nextCheckpoint = requireMongoCheckpoint(options.nextCheckpoint);
+    if (options.lastCheckpoint.checkpoint < nextCheckpoint.parameterChangesInvalidBefore) {
+      // Parameter compaction may have deleted parameter entries in the range we'd have to query
+      // to find the individual changed lookups. Invalidate all parameter buckets instead.
+      //
+      // The fence is committed before the first delete of a compaction pass, and captured in the
+      // same snapshot as the checkpoint, so a checkpoint that could miss a deleted entry always
+      // observes the fence as well.
+      return {
+        invalidateParameterBuckets: true,
+        updatedParameterLookups: new Set<string>()
+      };
+    }
+    // The query below runs at the checkpoint snapshot, which still sees entries deleted by a
+    // compaction pass that started after the snapshot.
+    return this.getParameterBucketChangesImpl({
+      lastCheckpoint: options.lastCheckpoint,
+      nextCheckpoint
+    });
   }
 
   private checkpointChangesCache = new LRUCache<
@@ -611,7 +664,10 @@ export abstract class MongoSyncBucketStorage
   });
 
   async getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<InternalCheckpointChanges> {
-    const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}`;
+    // The invalidation fence is part of the identity: the same checkpoint pair read before and
+    // after a compaction pass produces different results (specific lookups vs. invalidate-all).
+    const fence = requireMongoCheckpoint(options.nextCheckpoint).parameterChangesInvalidBefore;
+    const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}_${fence}`;
     const result = await this.checkpointChangesCache.fetch(key, { context: { options } });
     return result!;
   }
@@ -629,7 +685,19 @@ export abstract class MongoSyncBucketStorage
   }
 }
 
-class MongoReplicationCheckpoint implements ReplicationCheckpoint {
+/**
+ * We don't support any other constructions of ReplicationCheckpoint.
+ */
+function requireMongoCheckpoint(checkpoint: ReplicationCheckpoint): MongoReplicationCheckpoint {
+  if (!(checkpoint instanceof MongoReplicationCheckpoint)) {
+    throw new ServiceAssertionError(
+      `Checkpoint changes require a checkpoint from getCheckpointInternal(), got ${checkpoint.constructor.name}`
+    );
+  }
+  return checkpoint;
+}
+
+class MongoReplicationCheckpoint implements MongoSyncBucketStorageCheckpoint {
   #storage: MongoSyncBucketStorage;
 
   constructor(
@@ -637,7 +705,9 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
     public readonly checkpoint: InternalOpId,
     public readonly lsn: string | null,
     public snapshotTime: mongo.Timestamp,
-    public clusterTime: mongo.ClusterTime
+    public clusterTime: mongo.ClusterTime,
+    /** Captured in the same snapshot as the checkpoint. */
+    public readonly parameterChangesInvalidBefore: InternalOpId
   ) {
     this.#storage = storage;
   }
@@ -647,6 +717,10 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
   }
 }
 
+/**
+ * Used when no checkpoint has been persisted yet. This has no snapshot or invalidation fence, so
+ * it cannot be used for checkpoint change detection - see {@link requireMongoCheckpoint}.
+ */
 class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
   readonly checkpoint: InternalOpId = 0n;
   readonly lsn: string | null = null;

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -5,6 +5,8 @@ import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framewo
 import { storage } from '@powersync/service-core';
 import { VersionedPowerSyncMongo } from './db.js';
 
+const LOCK_DURATION_MS = 60 * 1000;
+
 /**
  * Manages a lock on a replication stream document, so that only one process
  * processes that replication stream at a time.
@@ -22,15 +24,20 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   ): Promise<MongoSyncRulesLock> {
     const lockId = crypto.randomBytes(8).toString('hex');
     const doc = await db.sync_rules.findOneAndUpdate(
-      { _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
       {
-        $set: {
-          lock: {
-            id: lockId,
-            expires_at: new Date(Date.now() + 60 * 1000)
+        _id: sync_rules.replicationStreamId,
+        $or: [{ lock: null }, { $expr: { $lt: ['$lock.expires_at', '$$NOW'] } }]
+      },
+      [
+        {
+          $set: {
+            lock: {
+              id: lockId,
+              expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS } }
+            }
           }
         }
-      },
+      ],
       {
         projection: { lock: 1 },
         returnDocument: 'before',
@@ -96,9 +103,15 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
         _id: this.sync_rules_id,
         'lock.id': this.lock_id
       },
-      {
-        $set: { 'lock.expires_at': new Date(Date.now() + 60 * 1000) }
-      },
+      [
+        {
+          $set: {
+            'lock.expires_at': {
+              $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS }
+            }
+          }
+        }
+      ],
       { returnDocument: 'after' }
     );
     if (result == null) {

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -1,29 +1,31 @@
-import { mongo } from '@powersync/lib-service-mongodb';
 import * as framework from '@powersync/lib-services-framework';
-import { GetCheckpointChangesOptions, InternalOpId, storage } from '@powersync/service-core';
+import { GetCheckpointChangesOptions, storage } from '@powersync/service-core';
 import { VersionedPowerSyncMongo } from './db.js';
 
 export type MongoCheckpointAPIOptions = {
   db: VersionedPowerSyncMongo;
-  mode: storage.WriteCheckpointMode;
-  sync_rules_id: number;
+  writeCheckpointMode: storage.WriteCheckpointModeConfig;
+  replicationStreamId: number;
 };
 
 export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
   readonly db: VersionedPowerSyncMongo;
+  readonly replicationStreamId: number;
+
   private _mode: storage.WriteCheckpointMode;
 
   constructor(options: MongoCheckpointAPIOptions) {
     this.db = options.db;
-    this._mode = options.mode;
+    this._mode = options.writeCheckpointMode.mode;
+    this.replicationStreamId = options.replicationStreamId;
   }
 
   get writeCheckpointMode() {
     return this._mode;
   }
 
-  setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
-    this._mode = mode;
+  setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    this._mode = config.mode;
   }
 
   async createManagedWriteCheckpoints(
@@ -247,6 +249,7 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
   }
 
   protected async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters) {
+    // Legacy V1/V2 implementation.
     const { user_id, sync_rules_id } = filters;
     const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
       user_id,
@@ -301,8 +304,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     };
   }
 
-  private async getCustomWriteCheckpointChanges(options: GetCheckpointChangesOptions) {
+  protected async getCustomWriteCheckpointChanges(options: GetCheckpointChangesOptions) {
     const limit = 1000;
+    // Legacy V1/V2 implementation.
     const changes = await this.db.custom_write_checkpoints
       .find(
         {
@@ -332,39 +336,3 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     };
   }
 }
-
-export async function batchCreateCustomWriteCheckpoints(
-  db: VersionedPowerSyncMongo,
-  session: mongo.ClientSession,
-  checkpoints: storage.CustomWriteCheckpointOptions[],
-  opId: InternalOpId
-): Promise<void> {
-  if (checkpoints.length == 0) {
-    return;
-  }
-
-  await db.custom_write_checkpoints.bulkWrite(
-    checkpoints.map((checkpointOptions) => {
-      const set: Record<string, unknown> = {
-        checkpoint: checkpointOptions.checkpoint,
-        sync_rules_id: checkpointOptions.sync_rules_id,
-        op_id: opId
-      };
-      if (checkpointOptions.checkpoint_requested_at != null) {
-        set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
-      }
-
-      return {
-        updateOne: {
-          filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
-          update: {
-            $set: set,
-            ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
-          },
-          upsert: true
-        }
-      };
-    }),
-    { session }
-  );
-}

package/src/storage/implementation/SyncRuleStateUpdate.ts

@@ -2,21 +2,28 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
 
 /**
- * Update pipeline to update replication stream status, covering all storage versions.
+ * Update pipeline to stop a deploying replication stream, covering all storage versions.
  *
  * Roughly equivalent to:
  *   $set: {
- *     state: state,
- *     'sync_configs.$[].state': state
+ *     state: STOP,
+ *     'sync_configs.$[].state': STOP
  *   }
  *
  * The difference is that this also handles v1 storage cases, where `sync_configs` is not present.
+ *
+ * This can be used for specific use cases:
+ * 1. Stop an existing PROCESSING stream, when it is being replaced by a new PROCESSING one.
+ * 2. Stop an existing ACTIVE or ERRORED stream, when it is being replaced by a new ACTIVE one.
+ *
+ * Do not use this when an ACTIVE stream transitions to ERRORED, and is being replaced by a new PROCESSING one.
+ * In that case, we need to maintain the ERRORED state, which this pipeline does not cover.
  */
-export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo.Document[] {
+export function stopReplicationStreamPipeline(): mongo.Document[] {
   return [
     {
       $set: {
-        state,
+        state: storage.SyncRuleState.STOP,
         sync_configs: {
           $cond: [
             { $isArray: '$sync_configs' },
@@ -25,7 +32,7 @@ export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo
                 input: '$sync_configs',
                 as: 'config',
                 in: {
-                  $mergeObjects: ['$$config', { state }]
+                  $mergeObjects: ['$$config', { state: storage.SyncRuleState.STOP }]
                 }
               }
             },

package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts

@@ -1,9 +1,31 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { InternalOpId } from '@powersync/service-core';
+import { InternalOpId, ReplicationCheckpoint } from '@powersync/service-core';
 import * as bson from 'bson';
 
-export interface MongoSyncBucketStorageCheckpoint {
+export interface MongoSyncBucketStorageCheckpoint extends ReplicationCheckpoint {
   checkpoint: InternalOpId;
   snapshotTime: bson.Timestamp;
   clusterTime: mongo.ClusterTime;
+
+  /**
+   * The stream's `parameter_compaction.checkpoint_changes_invalid_before` boundary, read in the
+   * same snapshot as {@link checkpoint} and {@link snapshotTime}.
+   *
+   * Parameter compaction may have deleted parameter entries below this boundary, so checkpoint
+   * change detection cannot enumerate individual lookup changes from a checkpoint below it.
+   *
+   * 0n for streams that have never been compacted.
+   */
+  parameterChangesInvalidBefore: InternalOpId;
+}
+
+/**
+ * MongoDB-specific version of GetCheckpointChangesOptions.
+ *
+ * The next checkpoint carries the snapshot and invalidation boundary that change detection must
+ * be evaluated against. Only `checkpoint` is used from the previous one.
+ */
+export interface MongoGetCheckpointChangesOptions {
+  lastCheckpoint: ReplicationCheckpoint;
+  nextCheckpoint: MongoSyncBucketStorageCheckpoint;
 }

package/src/storage/implementation/common/PersistedBatch.ts

@@ -65,6 +65,11 @@ export interface PersistedBatchOptions {
   logger?: Logger;
   objectStorage?: ObjectStorage;
   inlineThresholdBytes?: number;
+  /**
+   * Aborts in-flight object storage uploads when replication stops.
+   */
+  signal?: AbortSignal;
+  objectStorageUsageWriterId?: string;
 }
 
 /**
@@ -81,6 +86,8 @@ export abstract class PersistedBatch {
 
   protected readonly objectStorage?: ObjectStorage;
   protected readonly inlineThresholdBytes: number = DEFAULT_INLINE_THRESHOLD_BYTES;
+  protected readonly signal?: AbortSignal;
+  protected readonly objectStorageUsageWriterId?: string;
 
   /**
    * For debug logging only.
@@ -102,6 +109,8 @@ export abstract class PersistedBatch {
     this.currentSize = writtenSize;
     this.logger = options?.logger ?? defaultLogger;
     this.objectStorage = options?.objectStorage;
+    this.signal = options?.signal;
+    this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
     if (options?.inlineThresholdBytes != null) {
       this.inlineThresholdBytes = options.inlineThresholdBytes;
     }
@@ -124,7 +133,8 @@ export abstract class PersistedBatch {
       remaining_buckets.set(currentBucketKey(mapped), mapped);
     }
 
-    const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)));
+    const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
+    const dchecksum = BigInt(utils.hashDelete(subkey));
 
     for (const evaluated of options.evaluated) {
       const definitionId = this.getBucketDefinitionId(evaluated.source);
@@ -159,6 +169,7 @@ export abstract class PersistedBatch {
         bucket: evaluated.bucket,
         sourceTableId: options.table.id,
         sourceKey: options.sourceKey,
+        subkey,
         table: evaluated.table,
         rowId: evaluated.id,
         checksum: BigInt(checksum),
@@ -181,6 +192,7 @@ export abstract class PersistedBatch {
         op_id,
         sourceTableId: options.table.id,
         sourceKey: options.sourceKey,
+        subkey,
         table: bucket.table,
         rowId: bucket.id,
         checksum: dchecksum
@@ -217,6 +229,14 @@ export abstract class PersistedBatch {
   protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
   protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
 
+  /**
+   * New bucket operations persist protocol subkeys during replication. Older
+   * V1 documents may not have one, so sync reads retain a fallback.
+   */
+  protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string {
+    return replicaIdToSubkey(table, key);
+  }
+
   protected get bucketDataCount(): number {
     return this.bucketData.length;
   }
@@ -234,17 +254,45 @@ export abstract class PersistedBatch {
         bucket,
         lastOp: op_id,
         incrementCount: 1,
-        incrementBytes: bytes
+        incrementBytes: bytes,
+        incrementChunks: 0
       });
     }
   }
 
+  /**
+   * V3 persists operations in chunks. Keep this separate from incrementBucket:
+   * operation counts are known while evaluating rows, while chunk counts and
+   * exact persisted bytes are only known after the writer has chunked a flush.
+   */
+  protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) {
+    const key = `${definitionId ?? ''}:${bucket}`;
+    const existingState = this.bucketStates.get(key);
+    if (existingState != null) {
+      existingState.incrementChunks += 1;
+      existingState.incrementBytes += bytes;
+    }
+  }
+
+  /**
+   * V3's compactor calculates byte deltas from persisted chunk metadata.
+   * Replace the writer's per-operation estimate with those exact sizes before
+   * flushing the corresponding bucket state.
+   */
+  protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) {
+    const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
+    if (state != null) {
+      state.incrementBytes = 0;
+    }
+  }
+
   protected addBucketDataPut(options: {
     op_id: InternalOpId;
     bucketKey: BucketKey;
     bucket: string;
     sourceTableId: storage.SourceTable['id'];
     sourceKey: storage.ReplicaId;
+    subkey: string;
     table: string;
     rowId: string;
     checksum: bigint;
@@ -256,6 +304,7 @@ export abstract class PersistedBatch {
       op: 'PUT',
       source_table: mongoTableId(options.sourceTableId),
       source_key: options.sourceKey,
+      subkey: options.subkey,
       table: options.table,
       row_id: options.rowId,
       checksum: options.checksum,
@@ -268,6 +317,7 @@ export abstract class PersistedBatch {
     bucketKey: BucketKey;
     sourceTableId: storage.SourceTable['id'];
     sourceKey: storage.ReplicaId;
+    subkey: string;
     table: string;
     rowId: string;
     checksum: bigint;
@@ -278,6 +328,7 @@ export abstract class PersistedBatch {
       op: 'REMOVE',
       source_table: mongoTableId(options.sourceTableId),
       source_key: options.sourceKey,
+      subkey: options.subkey,
       table: options.table,
       row_id: options.rowId,
       checksum: options.checksum,
@@ -377,4 +428,5 @@ export interface BucketStateUpdate {
   lastOp: InternalOpId;
   incrementCount: number;
   incrementBytes: number;
+  incrementChunks: number;
 }