@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts

@@ -3,6 +3,8 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import {
   CheckpointChanges,
+  CompactInitialReplicationOptions,
+  CompactInitialReplicationResults,
   GetCheckpointChangesOptions,
   InternalOpId,
   internalToExternalOpId,
@@ -18,12 +20,20 @@ import * as bson from 'bson';
 import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
 import { MongoBucketStorage } from '../../MongoBucketStorage.js';
 import { BucketDataDoc } from '../common/BucketDataDoc.js';
-import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
+import {
+  MongoGetCheckpointChangesOptions,
+  MongoSyncBucketStorageCheckpoint
+} from '../common/MongoSyncBucketStorageCheckpoint.js';
 import { MongoChecksums } from '../MongoChecksums.js';
 import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
 import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
-import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
+import {
+  MongoCheckpointState,
+  MongoSyncBucketStorage,
+  MongoSyncBucketStorageOptions
+} from '../MongoSyncBucketStorage.js';
+import { MongoCheckpointAPIOptions } from '../MongoWriteCheckpointAPI.js';
 import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
 import {
   BucketDataDocumentV3,
@@ -36,10 +46,13 @@ import {
 import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
 import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
 import { MongoCompactorV3 } from './MongoCompactorV3.js';
+import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js';
 import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
+import { MongoWriteCheckpointAPIV3 } from './MongoWriteCheckpointAPIV3.js';
 import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
 import { ObjectStorage } from './object-storage/ObjectStorage.js';
 import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
+import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
 export interface MongoSyncBucketStorageContextV3 {
@@ -188,21 +201,39 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     });
   }
 
+  protected override createWriteCheckpointAPI(options: MongoCheckpointAPIOptions): MongoWriteCheckpointAPIV3 {
+    return new MongoWriteCheckpointAPIV3({
+      ...options,
+      db: this.db,
+      syncConfigMapping: () => this.singleSyncConfigMapping()
+    });
+  }
+
   createMongoCompactor(options: MongoCompactOptions): MongoCompactor {
     return new MongoCompactorV3(this, this.db, options);
   }
 
+  override async compactInitialReplication(
+    options: CompactInitialReplicationOptions
+  ): Promise<CompactInitialReplicationResults> {
+    this.logger.info(`Compacting chunks after initial replication...`);
+    const start = Date.now();
+    const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
+    const compactedBuckets = await this.createMongoCompactor({
+      ...options,
+      maxOpId,
+      compactChunksOnly: true,
+      logger: this.logger
+    }).compact();
+    this.logger.info(`Compacted chunks after initial replication in ${(Date.now() - start) / 1000}s`);
+    return { buckets: compactedBuckets };
+  }
+
   protected createMongoParameterCompactor(
     checkpoint: InternalOpId,
     options: storage.CompactOptions
   ): MongoParameterCompactor {
-    return new MongoParameterCompactor(this.db, this.replicationStreamId, checkpoint, options, () =>
-      this.db
-        .listParameterIndexCollections(this.replicationStreamId)
-        .then((collections) =>
-          collections.map((c) => c.collection as unknown as lib_mongo.mongo.Collection<lib_mongo.mongo.Document>)
-        )
-    );
+    return new MongoParameterCompactorV3(this.db, this.replicationStreamId, checkpoint, options);
   }
 
   protected async fetchPersistedOpHead(): Promise<InternalOpId | null> {
@@ -214,13 +245,14 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
   }
 
   protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
+    const batchOptions = this.writerBatchOptions(options);
     const doc = await this.syncRulesCollection.findOne(
       { _id: this.replicationStreamId },
       { projection: { resume_lsn: 1 } }
     );
 
     return new MongoBucketBatchV3({
-      ...this.writerBatchOptions(options),
+      ...batchOptions,
       // The stream-level replication position - per-config checkpoint LSNs are consistency
       // markers and do not affect where replication resumes.
       resumeFromLsn: doc?.resume_lsn ?? null,
@@ -228,16 +260,15 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     });
   }
 
-  protected async fetchCheckpointState(
-    session: mongo.ClientSession
-  ): Promise<{ checkpoint: bigint; lsn: string | null } | null> {
+  protected async fetchCheckpointState(session: mongo.ClientSession): Promise<MongoCheckpointState | null> {
     const doc = await this.syncRulesCollection.findOne(
       this.syncConfigMatch({
         state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
       }),
       {
         session,
-        projection: this.syncConfigProjection()
+        // The invalidation fence must be read in the same snapshot as the checkpoint.
+        projection: this.syncConfigProjection({ 'parameter_compaction.checkpoint_changes_invalid_before': 1 })
       }
     );
     // Checkpoints are served from the single active config. A PROCESSING config in the same
@@ -258,7 +289,10 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     }
     return {
       checkpoint: syncConfig.last_checkpoint ?? 0n,
-      lsn: syncConfig.last_checkpoint_lsn ?? null
+      lsn: syncConfig.last_checkpoint_lsn ?? null,
+      // Stream-level state: shared by all sync configs. Defaults to 0n for streams that have
+      // never been compacted.
+      parameterChangesInvalidBefore: doc?.parameter_compaction?.checkpoint_changes_invalid_before ?? 0n
     };
   }
 
@@ -311,7 +345,8 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
         },
         $unset: {
           resume_lsn: 1,
-          last_persisted_op: 1
+          last_persisted_op: 1,
+          parameter_compaction: 1
         }
       },
       {
@@ -376,6 +411,8 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
       await collection.drop();
     }
+    const usage = new ObjectStorageUsage(this.db, this.replicationStreamId);
+    await this.db.client.withSession((session) => session.withTransaction(() => usage.removeStream(session)));
     if (this.objectStorage) {
       const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
       await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
@@ -384,7 +421,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
       .pendingObjectStorageDeletes(this.replicationStreamId)
       .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
       .catch((error) => {
-        if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+        if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
           return;
         }
         throw error;
@@ -408,7 +445,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
       .bucketState(this.replicationStreamId)
       .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
       .catch((error) => {
-        if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+        if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
           return;
         }
         throw error;
@@ -420,11 +457,23 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
       .sourceTables(this.replicationStreamId)
       .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
       .catch((error) => {
+        if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
+          return;
+        }
+        throw error;
+      });
+  }
+
+  protected override async clearCustomCheckpointRequests(signal?: AbortSignal): Promise<void> {
+    for (const collection of await this.db.listCustomCheckpointRequestCollections(this.replicationStreamId)) {
+      signal?.throwIfAborted();
+      await collection.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }).catch((error) => {
         if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
           return;
         }
         throw error;
       });
+    }
   }
 
   async cleanupStoppedSyncConfigs(
@@ -449,7 +498,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
   }
 
   protected getParameterBucketChangesImpl(
-    options: GetCheckpointChangesOptions
+    options: MongoGetCheckpointChangesOptions
   ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
     return getParameterBucketChangesV3(this.versionContext, options);
   }
@@ -647,7 +696,7 @@ export async function* getBucketDataBatchV3(
     // Hydrate any operations from object storage.
     // In the future we can do this in a more pipelined fashion, but for now we hydrate
     // the entire batch at once.
-    await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal });
+    await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal, tracer: options?.tracer });
 
     let currentChunkSizeBytes = 0;
     let currentChunk: utils.SyncBucketData | null = null;
@@ -784,9 +833,17 @@ export async function getDataBucketChangesV3(
   };
 }
 
+/**
+ * Query the parameter entries changed between the two checkpoints, to determine which parameter
+ * lookups need to be re-evaluated.
+ *
+ * This runs at the next checkpoint's snapshot, so it still sees entries that parameter compaction
+ * deleted after that snapshot. Compaction that deleted entries before the snapshot is covered by
+ * the invalidation fence, checked before we get here.
+ */
 export async function getParameterBucketChangesV3(
   ctx: MongoSyncBucketStorageContextV3,
-  options: GetCheckpointChangesOptions
+  options: MongoGetCheckpointChangesOptions
 ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
   const limit = 1000;
   const indexIds = ctx.mapping.allParameterIndexIds();
@@ -816,28 +873,41 @@ export async function getParameterBucketChangesV3(
     }
   ];
   const [firstCollection, ...remainingCollections] = collections;
-  const parameterUpdates = await firstCollection.collection
-    .aggregate<{ lookup: bson.Binary; indexId: string }>(
-      [
-        ...pipelineForCollection(firstCollection.indexId),
-        ...remainingCollections.map((collection) => {
-          return {
-            $unionWith: {
-              coll: collection.collection.collectionName,
-              pipeline: pipelineForCollection(collection.indexId)
-            }
-          };
-        }),
+  const parameterUpdates = await ctx.db.client.withSession({ snapshot: true }, async (session) => {
+    setSessionSnapshotTime(session, options.nextCheckpoint.snapshotTime);
+    return await firstCollection.collection
+      .aggregate<{ lookup: bson.Binary; indexId: string }>(
+        [
+          ...pipelineForCollection(firstCollection.indexId),
+          ...remainingCollections.map((collection) => {
+            return {
+              $unionWith: {
+                coll: collection.collection.collectionName,
+                pipeline: pipelineForCollection(collection.indexId)
+              }
+            };
+          }),
+          {
+            $limit: limit + 1
+          }
+        ],
         {
-          $limit: limit + 1
+          session,
+          readConcern: 'snapshot',
+          batchSize: limit + 2,
+          maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
         }
-      ],
-      {
-        batchSize: limit + 2,
-        maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
-      }
-    )
-    .toArray();
+      )
+      .toArray()
+      .catch((e) => {
+        // Includes the case where the checkpoint snapshot has expired. Degrading to
+        // invalidateParameterBuckets would be safe in itself - it reads nothing - but it gains
+        // nothing: the caller responds to that by re-evaluating the parameter queries at this
+        // same snapshot, which fails too. The checkpoint has to be refetched instead, which is
+        // what the existing sync retry behavior does.
+        throw lib_mongo.mapQueryError(e, 'while querying parameter changes');
+      });
+  });
 
   const invalidateParameterUpdates = parameterUpdates.length > limit;
 

package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts

@@ -0,0 +1,111 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  CustomWriteCheckpointFilters,
+  GetCheckpointChangesOptions,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage
+} from '@powersync/service-core';
+import { EventDefinitionId } from '@powersync/service-sync-rules';
+import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from '../MongoWriteCheckpointAPI.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+
+export type MongoCheckpointAPIV3Options = Omit<MongoCheckpointAPIOptions, 'db'> & {
+  db: VersionedPowerSyncMongoV3;
+  syncConfigMapping: () => SingleSyncConfigBucketDefinitionMapping;
+};
+
+export class MongoWriteCheckpointAPIV3 extends MongoWriteCheckpointAPI {
+  declare db: VersionedPowerSyncMongoV3;
+
+  // Supplied via the constructor or the setWriteCheckpointMode setter.
+  private _customWriteCheckpointEventName: string | undefined;
+  private readonly syncConfigMapping: () => SingleSyncConfigBucketDefinitionMapping;
+
+  constructor(options: MongoCheckpointAPIV3Options) {
+    super(options);
+    this.syncConfigMapping = options.syncConfigMapping;
+    // Enforce the CUSTOM-requires-event-name invariant on construction too, not only setWriteCheckpointMode.
+    this._customWriteCheckpointEventName = this.requireCustomEventName(options.writeCheckpointMode);
+  }
+
+  protected resolveEventId(): EventDefinitionId {
+    const eventName = this._customWriteCheckpointEventName;
+    if (!eventName) {
+      throw new ServiceAssertionError(`No eventName has been supplied via setWriteCheckpointMode.`);
+    }
+    return this.syncConfigMapping().eventDefinitionIdByName(eventName);
+  }
+
+  protected override async lastCustomWriteCheckpoint(filters: CustomWriteCheckpointFilters): Promise<bigint | null> {
+    const { user_id } = filters;
+    const eventId = this.resolveEventId();
+
+    const lastWriteCheckpoint = await this.db
+      .customCheckpointRequests({
+        eventId,
+        replicationStreamId: this.replicationStreamId
+      })
+      .findOne({
+        user_id
+      });
+    return lastWriteCheckpoint?.checkpoint ?? null;
+  }
+
+  override setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    this._customWriteCheckpointEventName = this.requireCustomEventName(config);
+    super.setWriteCheckpointMode(config);
+  }
+
+  /**
+   * Validates and returns the event name for the given mode config. Shared by the constructor and
+   * {@link setWriteCheckpointMode} so both paths enforce that CUSTOM mode always names its checkpoint event.
+   */
+  private requireCustomEventName(config: storage.WriteCheckpointModeConfig): string | undefined {
+    if (config.mode != storage.WriteCheckpointMode.CUSTOM) {
+      return undefined;
+    }
+    if (!config.eventName) {
+      throw new ServiceAssertionError(`V3 incremental reprocessing requires an eventName to be supplied.`);
+    }
+    return config.eventName;
+  }
+
+  protected override async getCustomWriteCheckpointChanges(
+    options: GetCheckpointChangesOptions
+  ): Promise<{ invalidateWriteCheckpoints: boolean; updatedWriteCheckpoints: Map<string, bigint> }> {
+    const eventId = this.resolveEventId();
+
+    const limit = 1000;
+    const changes = await this.db
+      .customCheckpointRequests({
+        replicationStreamId: this.replicationStreamId,
+        eventId
+      })
+      .find(
+        {
+          op_id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }
+        },
+        {
+          limit: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
+          singleBatch: true
+        }
+      )
+      .toArray();
+    const invalidate = changes.length > limit;
+
+    const updatedWriteCheckpoints = new Map<string, bigint>();
+    if (!invalidate) {
+      for (let c of changes) {
+        updatedWriteCheckpoints.set(c.user_id, c.checkpoint);
+      }
+    }
+
+    return {
+      invalidateWriteCheckpoints: invalidate,
+      updatedWriteCheckpoints
+    };
+  }
+}