@powersync/service-module-mongodb-storage 0.23.2 → 0.24.0

package/src/storage/implementation/common/PersistedBatch.ts

@@ -1,43 +1,10 @@
-import { mongo } from '@powersync/lib-service-mongodb';
-import { BucketDataSource, BucketDefinitionId, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import { Logger } from '@powersync/lib-services-framework';
+import { InternalOpId, storage } from '@powersync/service-core';
+import { BucketDefinitionId, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-
-import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework';
-import { BucketDefinitionMapping, InternalOpId, storage, utils } from '@powersync/service-core';
-import { JSONBig } from '@powersync/service-jsonbig';
-import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
-import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
 import { MongoIdSequence } from '../MongoIdSequence.js';
-import { MongoWriteBatch } from '../MongoWriteBatch.js';
-import type { VersionedPowerSyncMongo } from '../db.js';
-import { TaggedBucketParameterDocument } from '../models.js';
-import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
-import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
 import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js';
 
-/**
- * Maximum size of operations we write in a single transaction.
- *
- * It's tricky to find the exact limit, but from experience, over 100MB
- * can cause an error:
- * > transaction is too large and will not fit in the storage engine cache
- *
- * Additionally, unbounded size here can balloon our memory usage in some edge
- * cases.
- *
- * When we reach this threshold, we commit the transaction and start a new one.
- */
-const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
-
-/**
- * Limit number of documents to write in a single transaction.
- *
- * This has an effect on error message size in some cases.
- */
-const MAX_TRANSACTION_DOC_COUNT = 2_000;
-
-export const DEFAULT_INLINE_THRESHOLD_BYTES = 16 * 1024;
-
 export interface SaveBucketDataOptions {
   op_seq: MongoIdSequence;
   sourceKey: storage.ReplicaId;
@@ -64,366 +31,21 @@ export interface UpsertCurrentDataOptions {
 
 export interface PersistedBatchOptions {
   logger?: Logger;
-  objectStorage?: ObjectStorage;
-  inlineThresholdBytes?: number;
-  /**
-   * Aborts in-flight object storage uploads when replication stops.
-   */
-  signal?: AbortSignal;
-  objectStorageUsageWriterId?: string;
 }
 
-/**
- * Keeps track of bulkwrite operations within a transaction.
- *
- * There may be multiple of these batches per transaction, but it may not span
- * multiple transactions.
- */
-export abstract class PersistedBatch {
-  logger: Logger;
-  bucketData: BucketDataDoc[] = [];
-  bucketParameters: TaggedBucketParameterDocument[] = [];
-  bucketStates: Map<string, BucketStateUpdate> = new Map();
-
-  protected readonly objectStorage?: ObjectStorage;
-  protected readonly inlineThresholdBytes: number = DEFAULT_INLINE_THRESHOLD_BYTES;
-  protected readonly signal?: AbortSignal;
-  protected readonly objectStorageUsageWriterId?: string;
-
-  /**
-   * For debug logging only.
-   */
-  debugLastOpId: InternalOpId | null = null;
-
-  /**
-   * Very rough estimate of transaction size.
-   */
-  currentSize = 0;
-
-  constructor(
-    protected readonly db: VersionedPowerSyncMongo,
-    protected readonly group_id: number,
-    protected readonly mapping: BucketDefinitionMapping,
-    writtenSize: number,
-    options?: PersistedBatchOptions
-  ) {
-    this.currentSize = writtenSize;
-    this.logger = options?.logger ?? defaultLogger;
-    this.objectStorage = options?.objectStorage;
-    this.signal = options?.signal;
-    this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
-    if (options?.inlineThresholdBytes != null) {
-      this.inlineThresholdBytes = options.inlineThresholdBytes;
-    }
-  }
-
-  saveBucketData(options: SaveBucketDataOptions) {
-    const remaining_buckets = new Map<string, SourceRecordBucketState>();
-    for (let bucket of options.before_buckets) {
-      const mapped: SourceRecordBucketState = {
-        bucket: bucket.bucket,
-        definitionId: this.checkDefinitionId(bucket.definitionId),
-        id: bucket.id,
-        table: bucket.table
-      };
-      if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId!)) {
-        // The bucket definition is not active anymore.
-        // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
-        continue;
-      }
-      remaining_buckets.set(currentBucketKey(mapped), mapped);
-    }
-
-    const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
-    const dchecksum = BigInt(utils.hashDelete(subkey));
-
-    for (const evaluated of options.evaluated) {
-      const definitionId = this.getBucketDefinitionId(evaluated.source);
-      const key = currentBucketKey({
-        definitionId: definitionId,
-        bucket: evaluated.bucket,
-        table: evaluated.table,
-        id: evaluated.id
-      });
-
-      const recordData = JSONBig.stringify(evaluated.data);
-      const checksum = utils.hashData(evaluated.table, evaluated.id, recordData);
-      if (recordData.length > MAX_ROW_SIZE) {
-        this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`);
-        continue;
-      }
-
-      remaining_buckets.delete(key);
-      const byteEstimate = recordData.length + 200;
-      this.currentSize += byteEstimate;
-
-      const op_id = options.op_seq.next();
-      this.debugLastOpId = op_id;
-
-      this.addBucketDataPut({
-        bucketKey: {
-          bucket: evaluated.bucket,
-          definitionId: definitionId,
-          replicationStreamId: this.group_id
-        },
-        op_id,
-        bucket: evaluated.bucket,
-        sourceTableId: options.table.id,
-        sourceKey: options.sourceKey,
-        subkey,
-        table: evaluated.table,
-        rowId: evaluated.id,
-        checksum: BigInt(checksum),
-        data: recordData
-      });
-      this.incrementBucket(definitionId, evaluated.bucket, op_id, byteEstimate);
-    }
-
-    for (let bucket of remaining_buckets.values()) {
-      const definitionId = bucket.definitionId!;
-      const op_id = options.op_seq.next();
-      this.debugLastOpId = op_id;
-
-      this.addBucketDataRemove({
-        bucketKey: {
-          replicationStreamId: this.group_id,
-          definitionId,
-          bucket: bucket.bucket
-        },
-        op_id,
-        sourceTableId: options.table.id,
-        sourceKey: options.sourceKey,
-        subkey,
-        table: bucket.table,
-        rowId: bucket.id,
-        checksum: dchecksum
-      });
-      this.currentSize += 200;
-      this.incrementBucket(definitionId, bucket.bucket, op_id, 200);
-    }
-  }
-
-  abstract saveParameterData(data: SaveParameterDataOptions): void;
-
-  abstract hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void;
-
-  abstract softDeleteCurrentData(
+/** Operations used by the shared writer; each storage version owns its lifecycle. */
+export interface PersistedBatch {
+  readonly currentSize: number;
+  saveBucketData(options: SaveBucketDataOptions): void;
+  saveParameterData(options: SaveParameterDataOptions): void;
+  hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void;
+  softDeleteCurrentData(
     sourceTableId: bson.ObjectId,
     replicaId: storage.ReplicaId,
     checkpointGreaterThan: bigint
   ): void;
-
-  abstract upsertCurrentData(values: UpsertCurrentDataOptions): void;
-
-  protected abstract get currentDataCount(): number;
-
-  protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
-
-  protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
-
-  protected abstract queueCurrentData(writes: MongoWriteBatch): void;
-
-  protected abstract queueBucketStates(writes: MongoWriteBatch): void;
-
-  protected abstract resetCurrentData(): void;
-
-  protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
-  protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
-
-  /**
-   * New bucket operations persist protocol subkeys during replication. Older
-   * V1 documents may not have one, so sync reads retain a fallback.
-   */
-  protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string {
-    return replicaIdToSubkey(table, key);
-  }
-
-  protected get bucketDataCount(): number {
-    return this.bucketData.length;
-  }
-
-  protected incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number) {
-    const key = `${definitionId ?? ''}:${bucket}`;
-    let existingState = this.bucketStates.get(key);
-    if (existingState) {
-      existingState.lastOp = op_id;
-      existingState.incrementCount += 1;
-      existingState.incrementBytes += bytes;
-    } else {
-      this.bucketStates.set(key, {
-        definitionId,
-        bucket,
-        lastOp: op_id,
-        incrementCount: 1,
-        incrementBytes: bytes,
-        incrementChunks: 0
-      });
-    }
-  }
-
-  /**
-   * V3 persists operations in chunks. Keep this separate from incrementBucket:
-   * operation counts are known while evaluating rows, while chunk counts and
-   * exact persisted bytes are only known after the writer has chunked a flush.
-   */
-  protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) {
-    const key = `${definitionId ?? ''}:${bucket}`;
-    const existingState = this.bucketStates.get(key);
-    if (existingState != null) {
-      existingState.incrementChunks += 1;
-      existingState.incrementBytes += bytes;
-    }
-  }
-
-  /**
-   * V3's compactor calculates byte deltas from persisted chunk metadata.
-   * Replace the writer's per-operation estimate with those exact sizes before
-   * flushing the corresponding bucket state.
-   */
-  protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) {
-    const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
-    if (state != null) {
-      state.incrementBytes = 0;
-    }
-  }
-
-  protected addBucketDataPut(options: {
-    op_id: InternalOpId;
-    bucketKey: BucketKey;
-    bucket: string;
-    sourceTableId: storage.SourceTable['id'];
-    sourceKey: storage.ReplicaId;
-    subkey: string;
-    table: string;
-    rowId: string;
-    checksum: bigint;
-    data: string;
-  }) {
-    this.bucketData.push({
-      bucketKey: options.bucketKey,
-      o: options.op_id,
-      op: 'PUT',
-      source_table: mongoTableId(options.sourceTableId),
-      source_key: options.sourceKey,
-      subkey: options.subkey,
-      table: options.table,
-      row_id: options.rowId,
-      checksum: options.checksum,
-      data: options.data
-    });
-  }
-
-  protected addBucketDataRemove(options: {
-    op_id: InternalOpId;
-    bucketKey: BucketKey;
-    sourceTableId: storage.SourceTable['id'];
-    sourceKey: storage.ReplicaId;
-    subkey: string;
-    table: string;
-    rowId: string;
-    checksum: bigint;
-  }) {
-    this.bucketData.push({
-      bucketKey: options.bucketKey,
-      o: options.op_id,
-      op: 'REMOVE',
-      source_table: mongoTableId(options.sourceTableId),
-      source_key: options.sourceKey,
-      subkey: options.subkey,
-      table: options.table,
-      row_id: options.rowId,
-      checksum: options.checksum,
-      data: null
-    });
-  }
-
-  shouldFlushTransaction() {
-    return (
-      this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
-      this.bucketDataCount >= MAX_TRANSACTION_DOC_COUNT ||
-      this.currentDataCount >= MAX_TRANSACTION_DOC_COUNT ||
-      this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT
-    );
-  }
-
-  async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
-    const startAt = performance.now();
-    let flushedSomething = false;
-    const writes = this.db.createWriteBatch(session, { ordered: false });
-    if (this.bucketDataCount > 0) {
-      flushedSomething = true;
-      await this.queueBucketData(writes);
-    }
-    if (this.bucketParameters.length > 0) {
-      flushedSomething = true;
-      this.queueBucketParameters(writes);
-    }
-    if (this.currentDataCount > 0) {
-      flushedSomething = true;
-      this.queueCurrentData(writes);
-    }
-
-    if (this.bucketStates.size > 0) {
-      flushedSomething = true;
-      this.queueBucketStates(writes);
-    }
-
-    await writes.execute();
-
-    if (flushedSomething) {
-      const duration = Math.round(performance.now() - startAt);
-      if (options?.oldestUncommittedChange != null) {
-        const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
-
-        this.logger.info(
-          `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
-            this.currentDataCount
-          } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
-          {
-            flushed: {
-              duration: duration,
-              size: this.currentSize,
-              bucket_ops_count: this.bucketDataCount,
-              parameter_indexes_count: this.bucketParameters.length,
-              source_records_count: this.currentDataCount,
-              replication_lag_seconds: replicationLag
-            }
-          }
-        );
-      } else {
-        this.logger.info(
-          `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
-            this.currentDataCount
-          } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
-          {
-            flushed: {
-              duration: duration,
-              size: this.currentSize,
-              bucket_ops_count: this.bucketDataCount,
-              parameter_indexes_count: this.bucketParameters.length,
-              source_records_count: this.currentDataCount
-            }
-          }
-        );
-      }
-    }
-
-    const stats = {
-      bucketDataCount: this.bucketDataCount,
-      parameterDataCount: this.bucketParameters.length,
-      currentDataCount: this.currentDataCount,
-      flushedAny: flushedSomething
-    };
-
-    this.bucketData = [];
-    this.bucketParameters = [];
-    this.resetCurrentData();
-    this.bucketStates.clear();
-    this.currentSize = 0;
-    this.debugLastOpId = null;
-
-    return stats;
-  }
+  upsertCurrentData(values: UpsertCurrentDataOptions): void;
+  shouldFlushTransaction(): boolean;
 }
 
 export interface BucketStateUpdate {

package/src/storage/implementation/common/PreparedPublication.ts

@@ -0,0 +1,49 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
+
+export interface PublicationStats {
+  bucketDataCount: number;
+  parameterDataCount: number;
+  currentDataCount: number;
+  flushedAny: boolean;
+}
+
+/**
+ * Sealed view of a v3/v4 batch. Preparation runs once, including on failure;
+ * publication creates fresh MongoDB writes on every attempt without clearing it.
+ *
+ * Scheduling metadata belongs to the caller: lease, selected config/mappings,
+ * input suffix, checkpoint intent and durable progress. Replay is valid only
+ * while that context and upload protection remain valid. A changed owner or
+ * config, or expired payload, must abandon this generation, never mutate it to
+ * fit a different context or overwrite its object paths. Publication also
+ * asserts that assigned IDs advance the freshly fenced persisted head.
+ *
+ * No application repair callbacks run here. The coordinator collects
+ * markRecordUnavailable records during evaluation and delivers them only after
+ * confirmed commit. Diagnostics run during evaluation.
+ * Before/after-flush hooks, source events and custom checkpoint consumption also
+ * belong to that coordinator, never to replayable writes. Only a confirmed
+ * commit may consume input, advance durable progress or notify readers.
+ */
+export interface PreparedPublication {
+  /**
+   * Explicitly start preparation after consuming assigned IDs. Sealing itself
+   * starts no I/O, so the coordinator can establish ownership before yielding.
+   * A rejected preparation is terminal for this generation; all started uploads
+   * must have settled before this promise rejects.
+   */
+  preparePayloads(): Promise<void>;
+  /**
+   * Requires completed readiness before starting the publication transaction.
+   * An optional batch supplies independent head/checkpoint writes to execute
+   * together with this group's data. This method owns execution of that batch;
+   * callers must not execute it again or include another group's dependent writes.
+   */
+  publish(
+    session: mongo.ClientSession,
+    options?: storage.BucketBatchCommitOptions,
+    additionalWrites?: MongoWriteBatch
+  ): Promise<PublicationStats>;
+}

package/src/storage/implementation/common/SourceRecordStore.ts

@@ -1,5 +1,4 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { Logger } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { BucketDefinitionId, EvaluatedParameters, EvaluatedRow, ParameterIndexId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
@@ -44,5 +43,4 @@ export interface SourceRecordStore {
     sourceTableId: bson.ObjectId,
     limit: number
   ): Promise<LoadedSourceRecord[]>;
-  postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise<void>;
 }