@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -26,30 +26,196 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     this._mode = mode;
   }
 
-  async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
+  async createManagedWriteCheckpoints(
+    checkpoints: storage.ManagedWriteCheckpointOptions[]
+  ): Promise<storage.CreateManagedWriteCheckpointsResult> {
     if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
       throw new framework.ServiceAssertionError(
         `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
       );
     }
 
-    const { user_id, heads: lsns } = checkpoint;
-    const doc = await this.db.write_checkpoints.findOneAndUpdate(
-      {
-        user_id: user_id
-      },
-      {
-        $set: {
-          lsns,
-          processed_at_lsn: null
+    const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints);
+    if (uniqueCheckpoints.length == 0) {
+      return { writeCheckpoints: new Map(), shouldAdvance: false };
+    }
+
+    let shouldAdvance = false;
+    const writeCheckpoints = new Map<string, bigint>();
+    const generatedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id == null);
+    const suppliedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id != null);
+
+    if (generatedCheckpoints.length == 1) {
+      // For the common case of a single checkpoint, we can do this in a single request.
+      const { user_id, heads: lsns } = generatedCheckpoints[0];
+      const doc = await this.db.write_checkpoints.findOneAndUpdate(
+        {
+          user_id
         },
-        $inc: {
-          client_id: 1n
-        }
-      },
-      { upsert: true, returnDocument: 'after' }
+        {
+          $set: {
+            lsns,
+            processed_at_lsn: null
+          },
+          // Unset rather than set to null, to keep the document out of the
+          // partial checkpoint_requested_at index used for retention cleanup.
+          $unset: {
+            checkpoint_requested_at: 1
+          },
+          $inc: {
+            client_id: 1n
+          }
+        },
+        { upsert: true, returnDocument: 'after' }
+      );
+
+      // Generated checkpoints always reset processed_at_lsn, so they are pending
+      // and the source marker must be forced.
+      shouldAdvance = true;
+      writeCheckpoints.set(doc!.user_id, doc!.client_id);
+    } else if (generatedCheckpoints.length > 1) {
+      // For more than one generated checkpoint, this gives a constant 2 requests.
+      await this.db.write_checkpoints.bulkWrite(
+        generatedCheckpoints.map(({ user_id, heads: lsns }) => ({
+          updateOne: {
+            filter: { user_id },
+            update: {
+              $set: {
+                lsns,
+                processed_at_lsn: null
+              },
+              $unset: {
+                checkpoint_requested_at: 1
+              },
+              $inc: {
+                client_id: 1n
+              }
+            },
+            upsert: true
+          }
+        }))
+      );
+
+      const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
+      const docs = await this.db.write_checkpoints
+        .find(
+          {
+            user_id: { $in: userIds }
+          },
+          {
+            projection: {
+              user_id: 1,
+              client_id: 1
+            }
+          }
+        )
+        .toArray();
+
+      shouldAdvance = true;
+      for (const doc of docs) {
+        writeCheckpoints.set(doc.user_id, doc.client_id);
+      }
+    }
+
+    if (suppliedCheckpoints.length > 0) {
+      const suppliedResult = await this.createSuppliedManagedWriteCheckpoints(suppliedCheckpoints);
+      shouldAdvance ||= suppliedResult.shouldAdvance;
+      for (const [userId, writeCheckpoint] of suppliedResult.writeCheckpoints) {
+        writeCheckpoints.set(userId, writeCheckpoint);
+      }
+    }
+
+    return { writeCheckpoints, shouldAdvance };
+  }
+
+  private async createSuppliedManagedWriteCheckpoints(checkpoints: storage.ManagedWriteCheckpointOptions[]) {
+    const checkpointRequestedAt = new Date();
+
+    await this.db.write_checkpoints.bulkWrite(
+      checkpoints.map((checkpoint) => {
+        const { user_id, heads: lsns } = checkpoint;
+        const checkpointRequestId = checkpoint.checkpoint_request_id!;
+
+        // Supplied request ids are monotonic. updateMany preserves the existing
+        // upsert behavior for first checkpoint requests, while also advancing
+        // every duplicate row for the user_id that is lower than this request id.
+        const shouldApplyRequestId = {
+          $or: [
+            { $eq: [{ $ifNull: ['$client_id', null] }, null] },
+            { $lt: ['$client_id', { $literal: checkpointRequestId }] }
+          ]
+        };
+        // Equal retries keep their original source head but refresh retention.
+        const shouldRefreshRequest = {
+          $or: [
+            { $eq: [{ $ifNull: ['$client_id', null] }, null] },
+            { $lte: ['$client_id', { $literal: checkpointRequestId }] }
+          ]
+        };
+
+        return {
+          updateMany: {
+            filter: { user_id },
+            update: [
+              {
+                $set: {
+                  user_id,
+                  client_id: {
+                    $cond: [shouldApplyRequestId, { $literal: checkpointRequestId }, '$client_id']
+                  },
+                  lsns: {
+                    $cond: [shouldApplyRequestId, { $literal: lsns }, '$lsns']
+                  },
+                  processed_at_lsn: {
+                    $cond: [shouldApplyRequestId, null, '$processed_at_lsn']
+                  },
+                  checkpoint_requested_at: {
+                    $cond: [shouldRefreshRequest, { $literal: checkpointRequestedAt }, '$checkpoint_requested_at']
+                  }
+                }
+              }
+            ],
+            upsert: true
+          }
+        };
+      })
     );
-    return doc!.client_id;
+
+    // Fetch the final ids separately so stale requests can still return the
+    // checkpoint currently stored. If duplicate rows already exist for a
+    // user_id, use the highest stored id instead of whichever document MongoDB
+    // happens to return first.
+    const userIds = checkpoints.map((checkpoint) => checkpoint.user_id);
+    const docs = await this.db.write_checkpoints
+      .find(
+        {
+          user_id: { $in: userIds }
+        },
+        {
+          projection: {
+            user_id: 1,
+            client_id: 1,
+            processed_at_lsn: 1
+          }
+        }
+      )
+      .toArray();
+    const maxDocsByUser = new Map<string, (typeof docs)[number]>();
+    for (const doc of docs) {
+      const existing = maxDocsByUser.get(doc.user_id);
+      if (
+        existing == null ||
+        doc.client_id > existing.client_id ||
+        (doc.client_id == existing.client_id && existing.processed_at_lsn != null && doc.processed_at_lsn == null)
+      ) {
+        maxDocsByUser.set(doc.user_id, doc);
+      }
+    }
+
+    return {
+      writeCheckpoints: new Map([...maxDocsByUser].map(([userId, doc]) => [userId, doc.client_id])),
+      shouldAdvance: [...maxDocsByUser.values()].some((doc) => doc.processed_at_lsn == null)
+    };
   }
 
   async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
@@ -178,19 +344,27 @@ export async function batchCreateCustomWriteCheckpoints(
   }
 
   await db.custom_write_checkpoints.bulkWrite(
-    checkpoints.map((checkpointOptions) => ({
-      updateOne: {
-        filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
-        update: {
-          $set: {
-            checkpoint: checkpointOptions.checkpoint,
-            sync_rules_id: checkpointOptions.sync_rules_id,
-            op_id: opId
-          }
-        },
-        upsert: true
+    checkpoints.map((checkpointOptions) => {
+      const set: Record<string, unknown> = {
+        checkpoint: checkpointOptions.checkpoint,
+        sync_rules_id: checkpointOptions.sync_rules_id,
+        op_id: opId
+      };
+      if (checkpointOptions.checkpoint_requested_at != null) {
+        set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
       }
-    })),
+
+      return {
+        updateOne: {
+          filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
+          update: {
+            $set: set,
+            ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
+          },
+          upsert: true
+        }
+      };
+    }),
     { session }
   );
 }

package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts

@@ -0,0 +1,9 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { InternalOpId } from '@powersync/service-core';
+import * as bson from 'bson';
+
+export interface MongoSyncBucketStorageCheckpoint {
+  checkpoint: InternalOpId;
+  snapshotTime: bson.Timestamp;
+  clusterTime: mongo.ClusterTime;
+}

package/src/storage/implementation/common/PersistedBatch.ts

@@ -3,14 +3,14 @@ import { BucketDataSource, BucketDefinitionId, EvaluatedParameters, EvaluatedRow
 import * as bson from 'bson';
 
 import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework';
-import { InternalOpId, storage, utils } from '@powersync/service-core';
+import { BucketDefinitionMapping, InternalOpId, storage, utils } from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
-import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
 import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
 import { MongoIdSequence } from '../MongoIdSequence.js';
 import type { VersionedPowerSyncMongo } from '../db.js';
 import { TaggedBucketParameterDocument } from '../models.js';
+import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
 import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
 import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js';
 
@@ -35,6 +35,8 @@ const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
  */
 const MAX_TRANSACTION_DOC_COUNT = 2_000;
 
+export const DEFAULT_INLINE_THRESHOLD_BYTES = 16 * 1024;
+
 export interface SaveBucketDataOptions {
   op_seq: MongoIdSequence;
   sourceKey: storage.ReplicaId;
@@ -61,6 +63,8 @@ export interface UpsertCurrentDataOptions {
 
 export interface PersistedBatchOptions {
   logger?: Logger;
+  objectStorage?: ObjectStorage;
+  inlineThresholdBytes?: number;
 }
 
 /**
@@ -75,6 +79,9 @@ export abstract class PersistedBatch {
   bucketParameters: TaggedBucketParameterDocument[] = [];
   bucketStates: Map<string, BucketStateUpdate> = new Map();
 
+  protected readonly objectStorage?: ObjectStorage;
+  protected readonly inlineThresholdBytes: number = DEFAULT_INLINE_THRESHOLD_BYTES;
+
   /**
    * For debug logging only.
    */
@@ -94,10 +101,14 @@ export abstract class PersistedBatch {
   ) {
     this.currentSize = writtenSize;
     this.logger = options?.logger ?? defaultLogger;
+    this.objectStorage = options?.objectStorage;
+    if (options?.inlineThresholdBytes != null) {
+      this.inlineThresholdBytes = options.inlineThresholdBytes;
+    }
   }
 
   saveBucketData(options: SaveBucketDataOptions) {
-    const remaining_buckets = new Map<string, SaveBucketDataOptions['before_buckets'][number]>();
+    const remaining_buckets = new Map<string, SourceRecordBucketState>();
     for (let bucket of options.before_buckets) {
       const mapped: SourceRecordBucketState = {
         bucket: bucket.bucket,
@@ -105,6 +116,11 @@ export abstract class PersistedBatch {
         id: bucket.id,
         table: bucket.table
       };
+      if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId!)) {
+        // The bucket definition is not active anymore.
+        // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
+        continue;
+      }
       remaining_buckets.set(currentBucketKey(mapped), mapped);
     }
 

package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts

@@ -67,7 +67,7 @@ export abstract class BaseVersionedPowerSyncMongo {
   }
 
   protected async listCollectionsByPrefix<T extends mongo.Document>(prefix: string): Promise<mongo.Collection<T>[]> {
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+    const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
 
     return collections
       .filter((collection) => collection.name.startsWith(prefix))

package/src/storage/implementation/createMongoSyncBucketStorage.ts

@@ -1,6 +1,6 @@
 import { storage } from '@powersync/service-core';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
 import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
 import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
 import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
@@ -11,15 +11,29 @@ export type { MongoSyncBucketStorage };
 
 export function createMongoSyncBucketStorage(
   factory: MongoBucketStorage,
-  group_id: number,
-  sync_rules: MongoPersistedSyncRulesContentV1,
-  slot_name: string,
+  replicationStreamId: number,
+  replicationStream: MongoPersistedReplicationStream,
+  replicationStreamName: string,
   writeCheckpointMode: storage.WriteCheckpointMode | undefined,
   options: MongoSyncBucketStorageOptions
 ): MongoSyncBucketStorage {
-  if (sync_rules.getStorageConfig().incrementalReprocessing) {
-    return new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  if (replicationStream.getStorageConfig().incrementalReprocessing) {
+    return new MongoSyncBucketStorageV3(
+      factory,
+      replicationStreamId,
+      replicationStream,
+      replicationStreamName,
+      writeCheckpointMode,
+      options
+    );
   }
 
-  return new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  return new MongoSyncBucketStorageV1(
+    factory,
+    replicationStreamId,
+    replicationStream,
+    replicationStreamName,
+    writeCheckpointMode,
+    options
+  );
 }

package/src/storage/implementation/db.ts

@@ -81,7 +81,7 @@ export class PowerSyncMongo {
   versioned(storageConfig: StorageConfig & { incrementalReprocessing: false }): VersionedPowerSyncMongoV1;
   versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo;
   versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo {
-    if (storageConfig.incrementalReprocessing) {
+    if (storageConfig.version >= storage.STORAGE_VERSION_3) {
       return new VersionedPowerSyncMongoV3(this, storageConfig);
     }
 
@@ -93,22 +93,18 @@ export class PowerSyncMongo {
    */
   async listBucketDataCollectionsV3(groupId?: number): Promise<mongo.Collection<BucketDataDocumentV3>[]> {
     const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
-
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<BucketDataDocumentV3>(collection.name));
+    return this.collectionsByPrefix<BucketDataDocumentV3>(prefix);
   }
 
   /**
    * Not safe for user-provided prefix - only for hardcoded values.
    */
-  private async collectionsByPrefix(prefix: string): Promise<mongo.Collection<never>[]> {
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+  private async collectionsByPrefix<T extends mongo.Document = never>(prefix: string): Promise<mongo.Collection<T>[]> {
+    const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
 
     return collections
       .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<never>(collection.name));
+      .map((collection) => this.db.collection<T>(collection.name));
   }
 
   /**
@@ -144,16 +140,12 @@ export class PowerSyncMongo {
   async listSourceTableCollections(
     replicationStreamId?: number
   ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
-    const filter =
-      replicationStreamId == null
-        ? { name: new RegExp('^source_table_') }
-        : { name: this.sourceTableCollectionName(replicationStreamId) };
-    const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId);
-    const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray();
-
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<CommonSourceTableDocument>(collection.name));
+    if (replicationStreamId == null) {
+      return this.collectionsByPrefix('source_table_');
+    }
+    const name = this.sourceTableCollectionName(replicationStreamId);
+    const collections = await this.db.listCollections({ name }, { nameOnly: true }).toArray();
+    return collections.map((c) => this.db.collection<CommonSourceTableDocument>(c.name));
   }
 
   /**
@@ -203,7 +195,14 @@ export class PowerSyncMongo {
    * This is used in a similar way to the Postgres NOTIFY functionality.
    */
   async notifyCheckpoint() {
-    await this.checkpoint_events.insertOne({} as any, { forceServerObjectId: true });
+    await this.checkpoint_events.insertOne({} as any, {
+      forceServerObjectId: true,
+      // No correctness issue if this write is missed / rolled back.
+      // It could cause some sync delays, that will resolve in at most a minute, but no consistency issues.
+      writeConcern: {
+        w: 1
+      }
+    });
   }
 
   /**

package/src/storage/implementation/models.ts

@@ -4,7 +4,7 @@ import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules
 import { event_types } from '@powersync/service-types';
 import * as bson from 'bson';
 import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
-import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js';
+import type { CurrentDataDocumentV3, RecordedLookupV3 } from './v3/models.js';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -78,6 +78,11 @@ export interface BucketDataProperties {
   row_id?: string;
   checksum: bigint;
   data: string | null;
+  /**
+   * V1-only.
+   *
+   * V3 stores this on the BucketDataDocumentV3 instead of the individual ops.
+   */
   target_op?: bigint | null;
 }
 
@@ -208,7 +213,7 @@ export interface SyncRuleDocumentBase {
   storage_version?: number;
 }
 
-export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint | null> {
+export interface SyncRuleCheckpointFields {
   /**
    * The last consistent checkpoint.
    *
@@ -225,16 +230,6 @@ export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint |
    * If set, no new checkpoints may be created < this value.
    */
   no_checkpoint_before: string | null;
-
-  /**
-   * Goes together with no_checkpoint_before.
-   *
-   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
-   * then the checkpoint must be equal to this keepalive_op.
-   *
-   * This is a string in V1, bigint in V3.
-   */
-  keepalive_op: TKeepaliveOp;
 }
 
 export interface StorageConfig extends storage.StorageVersionConfig {
@@ -283,6 +278,11 @@ export interface CustomWriteCheckpointDocument {
    * This is not unique - multiple write checkpoints can have the same op_id.
    */
   op_id?: InternalOpId;
+  /**
+   * Set when this checkpoint was created from a client-supplied checkpoint
+   * request rather than a persistent custom write checkpoint.
+   */
+  checkpoint_requested_at?: Date | null;
 }
 
 export interface WriteCheckpointDocument {
@@ -296,6 +296,12 @@ export interface WriteCheckpointDocument {
    * between two checkpoints.
    */
   processed_at_lsn: string | null;
+  /**
+   * Set when this checkpoint was created from a client-supplied checkpoint request
+   * rather than a server-generated id. A missing/null value means this is not a
+   * checkpoint request.
+   */
+  checkpoint_requested_at?: Date | null;
 }
 
 export interface InstanceDocument {
@@ -306,6 +312,6 @@ export interface InstanceDocument {
 export interface ClientConnectionDocument extends event_types.ClientConnection {}
 
 export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id'];
-export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3;
+export type CommonCurrentBucket = CurrentBucket;
 export type CommonCurrentLookup = bson.Binary | RecordedLookupV3;
-export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3;
+export type CommonSourceTableDocument = Pick<SourceTableDocumentV1, '_id' | 'snapshot_status'>;