@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts

@@ -0,0 +1,111 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  CustomWriteCheckpointFilters,
+  GetCheckpointChangesOptions,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage
+} from '@powersync/service-core';
+import { EventDefinitionId } from '@powersync/service-sync-rules';
+import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from '../MongoWriteCheckpointAPI.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+
+export type MongoCheckpointAPIV3Options = Omit<MongoCheckpointAPIOptions, 'db'> & {
+  db: VersionedPowerSyncMongoV3;
+  syncConfigMapping: () => SingleSyncConfigBucketDefinitionMapping;
+};
+
+export class MongoWriteCheckpointAPIV3 extends MongoWriteCheckpointAPI {
+  declare db: VersionedPowerSyncMongoV3;
+
+  // Supplied via the constructor or the setWriteCheckpointMode setter.
+  private _customWriteCheckpointEventName: string | undefined;
+  private readonly syncConfigMapping: () => SingleSyncConfigBucketDefinitionMapping;
+
+  constructor(options: MongoCheckpointAPIV3Options) {
+    super(options);
+    this.syncConfigMapping = options.syncConfigMapping;
+    // Enforce the CUSTOM-requires-event-name invariant on construction too, not only setWriteCheckpointMode.
+    this._customWriteCheckpointEventName = this.requireCustomEventName(options.writeCheckpointMode);
+  }
+
+  protected resolveEventId(): EventDefinitionId {
+    const eventName = this._customWriteCheckpointEventName;
+    if (!eventName) {
+      throw new ServiceAssertionError(`No eventName has been supplied via setWriteCheckpointMode.`);
+    }
+    return this.syncConfigMapping().eventDefinitionIdByName(eventName);
+  }
+
+  protected override async lastCustomWriteCheckpoint(filters: CustomWriteCheckpointFilters): Promise<bigint | null> {
+    const { user_id } = filters;
+    const eventId = this.resolveEventId();
+
+    const lastWriteCheckpoint = await this.db
+      .customCheckpointRequests({
+        eventId,
+        replicationStreamId: this.replicationStreamId
+      })
+      .findOne({
+        user_id
+      });
+    return lastWriteCheckpoint?.checkpoint ?? null;
+  }
+
+  override setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    this._customWriteCheckpointEventName = this.requireCustomEventName(config);
+    super.setWriteCheckpointMode(config);
+  }
+
+  /**
+   * Validates and returns the event name for the given mode config. Shared by the constructor and
+   * {@link setWriteCheckpointMode} so both paths enforce that CUSTOM mode always names its checkpoint event.
+   */
+  private requireCustomEventName(config: storage.WriteCheckpointModeConfig): string | undefined {
+    if (config.mode != storage.WriteCheckpointMode.CUSTOM) {
+      return undefined;
+    }
+    if (!config.eventName) {
+      throw new ServiceAssertionError(`V3 incremental reprocessing requires an eventName to be supplied.`);
+    }
+    return config.eventName;
+  }
+
+  protected override async getCustomWriteCheckpointChanges(
+    options: GetCheckpointChangesOptions
+  ): Promise<{ invalidateWriteCheckpoints: boolean; updatedWriteCheckpoints: Map<string, bigint> }> {
+    const eventId = this.resolveEventId();
+
+    const limit = 1000;
+    const changes = await this.db
+      .customCheckpointRequests({
+        replicationStreamId: this.replicationStreamId,
+        eventId
+      })
+      .find(
+        {
+          op_id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }
+        },
+        {
+          limit: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
+          singleBatch: true
+        }
+      )
+      .toArray();
+    const invalidate = changes.length > limit;
+
+    const updatedWriteCheckpoints = new Map<string, bigint>();
+    if (!invalidate) {
+      for (let c of changes) {
+        updatedWriteCheckpoints.set(c.user_id, c.checkpoint);
+      }
+    }
+
+    return {
+      invalidateWriteCheckpoints: invalidate,
+      updatedWriteCheckpoints
+    };
+  }
+}

package/src/storage/implementation/v3/PersistedBatchV3.ts

@@ -1,69 +1,99 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { InternalOpId, storage } from '@powersync/service-core';
+import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
 import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { mongoTableId } from '../../../utils/util.js';
+import { BucketDataDoc } from '../common/BucketDataDoc.js';
 import {
   BucketStateUpdate,
   PersistedBatch,
+  PersistedBatchOptions,
   SaveParameterDataOptions,
   UpsertCurrentDataOptions
 } from '../common/PersistedBatch.js';
-import { SourceTableKey } from '../models.js';
+import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
+import { serializeBucketData } from './bucket-format.js';
+import { chunkBucketData } from './chunking.js';
+import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js';
 import {
-  BucketParameterDocumentV3,
+  BucketDataDocumentV3,
   BucketStateDocumentV3,
   CurrentDataDocumentV3,
-  serializeBucketDataV3,
+  serializeParameterLookup,
   SourceTableDocumentV3,
-  taggedBucketParameterDocumentToV3
+  taggedBucketParameterDocumentToTagged
 } from './models.js';
-import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
+import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
+import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
 export class PersistedBatchV3 extends PersistedBatch {
-  declare protected readonly db: VersionedPowerSyncMongoV3;
-
   currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
   sourceTablePendingDeletes = new Map<string, InternalOpId>();
+  protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
+  protected readonly objectStorageUsage?: ObjectStorageUsage;
+
+  declare protected readonly db: VersionedPowerSyncMongoV3;
+
+  constructor(
+    db: VersionedPowerSyncMongoV3,
+    group_id: number,
+    mapping: BucketDefinitionMapping,
+    writtenSize: number,
+    options?: PersistedBatchOptions
+  ) {
+    super(db, group_id, mapping, writtenSize, options);
+    if (this.objectStorage) {
+      this.objectStorageLifecycle = new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage);
+      this.objectStorageUsage = new ObjectStorageUsage(this.db, this.group_id, this.objectStorageUsageWriterId);
+    }
+  }
+
+  // Abstract override from PersistedBatch (V3-specific error message)
 
-  protected checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
+  protected override checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
     if (definitionId == null) {
-      // This is required for V3 storage.
       throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
     }
     return definitionId;
   }
 
+  // Concrete implementations from PersistedBatchShared
+
   protected getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId {
     return this.mapping.bucketSourceId(bucketSource);
   }
 
   saveParameterData(data: SaveParameterDataOptions) {
     const { sourceTable, sourceKey, evaluated } = data;
-    const remaining_lookups = new Map<string, SaveParameterDataOptions['existing_lookups'][number]>();
+    const remaining_lookups = new Map<string, SourceRecordLookupState>();
 
     for (let lookup of data.existing_lookups) {
       if (lookup.indexId == null) {
-        throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
+      }
+      if (sourceTable.parameterLookupSourceIds != null && !sourceTable.parameterLookupSourceIds.has(lookup.indexId)) {
+        // The parameter index is not active anymore.
+        // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
+        continue;
       }
       remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup);
     }
 
     for (let result of evaluated) {
       const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source);
-      const binLookup = serializeParameterLookupV3(result.lookup);
+      const binLookup = serializeParameterLookup(result.lookup);
       remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`);
 
       const op_id = data.op_seq.next();
       this.debugLastOpId = op_id;
-      const values: BucketParameterDocumentV3 = {
+      const values = {
         _id: op_id,
         key: {
           t: mongoTableId(sourceTable.id),
           k: sourceKey
-        } satisfies SourceTableKey,
+        },
         lookup: binLookup,
         bucket_parameters: result.bucketParameters
       };
@@ -80,14 +110,14 @@ export class PersistedBatchV3 extends PersistedBatch {
       this.debugLastOpId = op_id;
       const indexId = lookup.indexId;
       if (indexId == null) {
-        throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
       }
-      const values: BucketParameterDocumentV3 = {
+      const values = {
         _id: op_id,
         key: {
           t: mongoTableId(sourceTable.id),
           k: sourceKey
-        } satisfies SourceTableKey,
+        },
         lookup: lookup.lookup,
         bucket_parameters: []
       };
@@ -125,8 +155,8 @@ export class PersistedBatchV3 extends PersistedBatch {
           update: {
             $set: {
               data: null,
-              buckets: [] as CurrentDataDocumentV3['buckets'],
-              lookups: [] as CurrentDataDocumentV3['lookups'],
+              buckets: [],
+              lookups: [],
               pending_delete: checkpointGreaterThan
             }
           },
@@ -146,7 +176,7 @@ export class PersistedBatchV3 extends PersistedBatch {
   upsertCurrentData(values: UpsertCurrentDataOptions) {
     const buckets = values.buckets.map((bucket) => {
       if (bucket.definitionId == null) {
-        throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected bucket when incrementalReprocessing is enabled');
       }
       return {
         def: bucket.definitionId,
@@ -157,7 +187,7 @@ export class PersistedBatchV3 extends PersistedBatch {
     });
     const lookups = values.lookups.map((lookup) => {
       if (lookup.indexId == null) {
-        throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
       }
       return {
         i: lookup.indexId,
@@ -189,26 +219,93 @@ export class PersistedBatchV3 extends PersistedBatch {
     return this.currentData.length;
   }
 
+  // Flush methods
+
   protected async flushBucketData(session: mongo.ClientSession) {
-    const operationsByDefinition = new Map<BucketDefinitionId, typeof this.bucketData>();
+    const operationsByDefinition = new Map<BucketDefinitionId, BucketDataDoc[]>();
     for (const document of this.bucketData) {
       const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
       existing.push(document);
       operationsByDefinition.set(document.bucketKey.definitionId, existing);
     }
 
-    for (const [definitionId, documents] of operationsByDefinition.entries()) {
-      await this.db.bucketDataV3(this.group_id, definitionId).bulkWrite(
-        documents.map((document) => ({
-          insertOne: {
-            document: serializeBucketDataV3(document)
+    const plans = Array.from(operationsByDefinition, ([definitionId, documents]) => {
+      const operationsByBucket = Map.groupBy(documents, (document) => document.bucketKey.bucket);
+      const lifecycle = this.objectStorageLifecycle;
+      type BucketDataInsert = { insertOne: { document: BucketDataDocumentV3 } };
+      const createInserts: (() => Promise<BucketDataInsert>)[] = [];
+
+      for (const [bucket, ops] of operationsByBucket) {
+        this.resetBucketPersistedBytes(definitionId, bucket);
+        for (const chunk of chunkBucketData(ops)) {
+          const serialized = serializeBucketData(bucket, chunk);
+          this.incrementBucketPersistedChunk(definitionId, bucket, serialized.size);
+          if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) {
+            createInserts.push(async () => ({
+              insertOne: {
+                document: serialized
+              }
+            }));
+            continue;
           }
-        })),
-        {
-          session,
-          ordered: false
+
+          createInserts.push(async () => {
+            const minOp = chunk[0].o;
+            const maxOp = chunk[chunk.length - 1].o;
+            const { ops: bucketOps, ...metadata } = serialized;
+            const path = lifecycle.allocatePath(definitionId, bucket, minOp, maxOp);
+            const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!, { signal: this.signal });
+            return {
+              insertOne: {
+                document: {
+                  ...metadata,
+                  storage_ref: {
+                    path,
+                    file_size: fileSize
+                  }
+                }
+              }
+            };
+          });
         }
+      }
+
+      return { definitionId, createInserts };
+    });
+
+    const createAllInserts = () =>
+      Promise.all(
+        plans.map(async ({ definitionId, createInserts }) => ({
+          definitionId,
+          inserts: await Promise.all(createInserts.map((createInsert) => createInsert()))
+        }))
       );
+
+    // S3ObjectStorage applies one shared concurrency limit across all callers,
+    // so replication can schedule its uploads together without creating a
+    // separate limiter here.
+    const writes = await createAllInserts();
+
+    const usageDeltas = this.objectStorageUsage == null ? undefined : new Map<BucketDefinitionId, bigint>();
+    for (const { definitionId, inserts } of writes) {
+      if (usageDeltas != null) {
+        const delta = inserts.reduce(
+          (sum, operation) => sum + ObjectStorageUsage.bytes(operation.insertOne.document),
+          0n
+        );
+        if (delta !== 0n) {
+          usageDeltas.set(definitionId, delta);
+        }
+      }
+      if (inserts.length > 0) {
+        await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
+          session,
+          ordered: false
+        });
+      }
+    }
+    if (usageDeltas != null) {
+      await this.objectStorageUsage!.applyDeltas(usageDeltas, session);
     }
   }
 
@@ -221,10 +318,10 @@ export class PersistedBatchV3 extends PersistedBatch {
     }
 
     for (const [indexId, documents] of operationsByIndex.entries()) {
-      await this.db.parameterIndexV3(this.group_id, indexId).bulkWrite(
+      await this.db.parameterIndex(this.group_id, indexId).bulkWrite(
         documents.map((document) => ({
           insertOne: {
-            document: taggedBucketParameterDocumentToV3(document)
+            document: taggedBucketParameterDocumentToTagged(document)
           }
         })),
         {
@@ -260,12 +357,12 @@ export class PersistedBatchV3 extends PersistedBatch {
     });
 
     if (sourceTableUpdates.length > 0) {
-      await this.db.sourceTablesV3(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
+      await this.db.sourceTables(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
     }
 
     for (const operations of operationsBySourceTable.values()) {
       const sourceTableId = operations[0]!.sourceTableId;
-      await this.db.sourceRecordsV3(this.group_id, sourceTableId).bulkWrite(
+      await this.db.sourceRecords(this.group_id, sourceTableId).bulkWrite(
         operations.map((entry) => entry.operation),
         {
           session,
@@ -276,7 +373,7 @@ export class PersistedBatchV3 extends PersistedBatch {
   }
 
   protected async flushBucketStates(session: mongo.ClientSession) {
-    await this.db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
+    await this.db.bucketState(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
       session,
       ordered: false
     });
@@ -300,15 +397,42 @@ export class PersistedBatchV3 extends PersistedBatch {
               b: state.bucket
             }
           },
-          update: {
-            $set: {
-              last_op: state.lastOp
-            },
-            $inc: {
-              'estimate_since_compact.count': state.incrementCount,
-              'estimate_since_compact.bytes': state.incrementBytes
+          // A pipeline update makes initialisation and scheduling one atomic
+          // writer operation. In particular, a later write cannot move an
+          // already-due compact check into the future.
+          update: [
+            {
+              $set: {
+                last_op: state.lastOp,
+                bucket_stats: {
+                  count: { $add: [{ $ifNull: ['$bucket_stats.count', 0] }, state.incrementCount] },
+                  bytes: { $add: [{ $ifNull: ['$bucket_stats.bytes', 0n] }, BigInt(state.incrementBytes)] },
+                  chunks: { $add: [{ $ifNull: ['$bucket_stats.chunks', 0] }, state.incrementChunks] }
+                },
+                first_uncompacted_write: { $ifNull: ['$first_uncompacted_write', '$$NOW'] },
+                next_compact_check: {
+                  $let: {
+                    vars: {
+                      requested: {
+                        $dateAdd: {
+                          startDate: '$$NOW',
+                          unit: 'millisecond',
+                          amount: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS
+                        }
+                      }
+                    },
+                    in: {
+                      $cond: [
+                        { $lt: [{ $ifNull: ['$next_compact_check', '$$requested'] }, '$$requested'] },
+                        '$next_compact_check',
+                        '$$requested'
+                      ]
+                    }
+                  }
+                }
+              }
             }
-          },
+          ],
           upsert: true
         }
       } satisfies mongo.AnyBulkWriteOperation<BucketStateDocumentV3>;

package/src/storage/implementation/v3/SourceRecordStoreV3.ts

@@ -1,21 +1,19 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
+import { BucketDefinitionMapping, storage } from '@powersync/service-core';
 import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
-import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
 import { cacheKey } from '../OperationBatch.js';
 import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js';
-import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
-import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js';
+import { CurrentDataDocumentV3, serializeParameterLookup, SourceTableDocumentV3 } from './models.js';
 
 export class SourceRecordStoreV3 implements SourceRecordStore {
   constructor(
     private readonly db: VersionedPowerSyncMongoV3,
-    private readonly groupId: number,
+    private readonly replicationStreamId: number,
     private readonly mapping: BucketDefinitionMapping
   ) {}
 
@@ -31,7 +29,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
   mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'] {
     return paramEvaluated.map((entry) => ({
       indexId: this.mapping.parameterLookupId(entry.lookup.source),
-      lookup: serializeParameterLookupV3(entry.lookup)
+      lookup: serializeParameterLookup(entry.lookup)
     }));
   }
 
@@ -67,7 +65,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
         _id: { $in: replicaIds as any[] }
       } as unknown as mongo.Filter<CurrentDataDocumentV3>;
       const sizeCursor: mongo.AggregationCursor<CurrentDataDocumentV3 & { size: number }> = this.db
-        .sourceRecordsV3(this.groupId, sourceTableId)
+        .sourceRecords(this.replicationStreamId, sourceTableId)
         .aggregate(
           [
             {
@@ -100,7 +98,9 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
       const filter = {
         _id: { $in: replicaIds as any[] }
       } as unknown as mongo.Filter<CurrentDataDocumentV3>;
-      const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(filter, { session, projection });
+      const cursor = this.db
+        .sourceRecords(this.replicationStreamId, sourceTableId)
+        .find(filter, { session, projection });
       for await (const doc of cursor.stream()) {
         const loaded = this.createLoadedDocument(
           sourceTableId,
@@ -120,7 +120,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     sourceTableId: bson.ObjectId,
     limit: number
   ): Promise<LoadedSourceRecord[]> {
-    const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(
+    const cursor = this.db.sourceRecords(this.replicationStreamId, sourceTableId).find(
       {
         pending_delete: { $exists: false }
       },
@@ -145,7 +145,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     // ones have dirty deletes in source_tables.
 
     const dirtySourceTables = await this.db
-      .sourceTablesV3(this.groupId)
+      .sourceTables(this.replicationStreamId)
       .find(
         {
           latest_pending_delete: { $exists: true }
@@ -159,7 +159,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     let deletedCount = 0;
     const sourceTableUpdates: mongo.AnyBulkWriteOperation<SourceTableDocumentV3>[] = [];
     for (const sourceTable of dirtySourceTables) {
-      const collection = this.db.sourceRecordsV3(this.groupId, sourceTable._id);
+      const collection = this.db.sourceRecords(this.replicationStreamId, sourceTable._id);
       const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger);
       deletedCount += result.deletedCount;
 
@@ -182,7 +182,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     }
 
     if (sourceTableUpdates.length > 0) {
-      await this.db.sourceTablesV3(this.groupId).bulkWrite(sourceTableUpdates, { ordered: false });
+      await this.db.sourceTables(this.replicationStreamId).bulkWrite(sourceTableUpdates, { ordered: false });
     }
     if (deletedCount > 0) {
       logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`);