@powersync/service-module-mongodb-storage 0.23.2 → 0.24.0

package/src/storage/implementation/v3/SourceRecordStoreV3.ts

@@ -5,11 +5,23 @@ import { BucketDefinitionMapping, storage } from '@powersync/service-core';
 import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
+import { REPLICATION_READ_OPTIONS, REPLICATION_WRITE_OPTIONS } from '../MongoReplicationOptions.js';
 import { cacheKey } from '../OperationBatch.js';
 import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 import { CurrentDataDocumentV3, serializeParameterLookup, SourceTableDocumentV3 } from './models.js';
 
+/**
+ * Source-record access for v3/v4 preparation and post-commit cleanup.
+ *
+ * The loadSizes(), loadDocuments() and loadTruncateBatch() methods are preparation
+ * reads: callers must hold the lease's writer mutex and use its shared causal
+ * session outside any transaction. These reads always request primary/majority
+ * consistency, so they observe earlier writes in that session. Keep the mutex
+ * held through publication to prevent another local writer from changing the
+ * source records between preparation and commit. Publication transactions write
+ * the prepared result without calling these read methods again.
+ */
 export class SourceRecordStoreV3 implements SourceRecordStore {
   constructor(
     private readonly db: VersionedPowerSyncMongoV3,
@@ -60,7 +72,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
 
   async loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise<Map<string, number>> {
     const sizes = new Map<string, number>();
-    for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) {
+    for (const { sourceTableId, replicaIds } of this.groupEntries(entries)) {
       const filter = {
         _id: { $in: replicaIds as any[] }
       } as unknown as mongo.Filter<CurrentDataDocumentV3>;
@@ -78,7 +90,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
               }
             }
           ],
-          { session }
+          { session, ...REPLICATION_READ_OPTIONS }
         );
       for await (const doc of sizeCursor.stream()) {
         sizes.set(cacheKey(sourceTableId, doc._id), doc.size);
@@ -94,13 +106,15 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
   ): Promise<Map<string, LoadedSourceRecord>> {
     const documents = new Map<string, LoadedSourceRecord>();
     const projection = idsOnly ? { _id: 1 } : undefined;
-    for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) {
+    for (const { sourceTableId, replicaIds } of this.groupEntries(entries)) {
       const filter = {
         _id: { $in: replicaIds as any[] }
       } as unknown as mongo.Filter<CurrentDataDocumentV3>;
-      const cursor = this.db
-        .sourceRecords(this.replicationStreamId, sourceTableId)
-        .find(filter, { session, projection });
+      const cursor = this.db.sourceRecords(this.replicationStreamId, sourceTableId).find(filter, {
+        session,
+        projection,
+        ...REPLICATION_READ_OPTIONS
+      });
       for await (const doc of cursor.stream()) {
         const loaded = this.createLoadedDocument(
           sourceTableId,
@@ -131,7 +145,8 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
           lookups: 1
         },
         limit,
-        session
+        session,
+        ...REPLICATION_READ_OPTIONS
       }
     );
     return (await cursor.toArray()).map((doc) =>
@@ -182,7 +197,9 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     }
 
     if (sourceTableUpdates.length > 0) {
-      await this.db.sourceTables(this.replicationStreamId).bulkWrite(sourceTableUpdates, { ordered: false });
+      await this.db
+        .sourceTables(this.replicationStreamId)
+        .bulkWrite(sourceTableUpdates, { ordered: false, ...REPLICATION_WRITE_OPTIONS });
     }
     if (deletedCount > 0) {
       logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`);
@@ -195,6 +212,8 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     lastCheckpoint: bigint,
     logger: Logger
   ) {
+    // A previous owner's in-flight cleanup must commute with a new
+    // publisher: live upserts clear pending_delete and never match.
     return retryOnMongoMaxTimeMSExpired(
       () =>
         collection.deleteMany(
@@ -202,7 +221,8 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
             pending_delete: { $exists: true, $lte: lastCheckpoint }
           },
           {
-            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
+            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+            ...REPLICATION_WRITE_OPTIONS
           }
         ),
       {
@@ -214,13 +234,23 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     );
   }
 
-  private groupEntries(entries: SourceRecordLookupEntry[]): Map<bson.ObjectId, storage.ReplicaId[]> {
-    const grouped = new Map<bson.ObjectId, storage.ReplicaId[]>();
+  /** Group table IDs by value and deduplicate replica IDs, preserving first-seen order. */
+  private groupEntries(
+    entries: SourceRecordLookupEntry[]
+  ): { sourceTableId: bson.ObjectId; replicaIds: storage.ReplicaId[] }[] {
+    const grouped = new Map<string, { sourceTableId: bson.ObjectId; replicas: Map<string, storage.ReplicaId> }>();
     for (const entry of entries) {
-      const existing = grouped.get(entry.sourceTableId) ?? [];
-      existing.push(entry.replicaId);
-      grouped.set(entry.sourceTableId, existing);
+      const key = entry.sourceTableId.toHexString();
+      let group = grouped.get(key);
+      if (group == null) {
+        group = { sourceTableId: entry.sourceTableId, replicas: new Map() };
+        grouped.set(key, group);
+      }
+      group.replicas.set(cacheKey(entry.sourceTableId, entry.replicaId), entry.replicaId);
     }
-    return grouped;
+    return [...grouped.values()].map(({ sourceTableId, replicas }) => ({
+      sourceTableId,
+      replicaIds: [...replicas.values()]
+    }));
   }
 }

package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts

@@ -12,6 +12,15 @@ export class BucketDataObjectStorage {
     options?: ObjectStorageOperationOptions
   ): Promise<{ fileSize: number }> {
     const bsonBuffer = bson.serialize({ ops });
+    return this.storeSerialized(path, bsonBuffer, options);
+  }
+
+  /** Store an already sealed BSON { ops } payload without serializing it again. */
+  async storeSerialized(
+    path: string,
+    bsonBuffer: Uint8Array,
+    options?: ObjectStorageOperationOptions
+  ): Promise<{ fileSize: number }> {
     await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null }, options);
     return { fileSize: bsonBuffer.byteLength };
   }

package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts

@@ -1,6 +1,8 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
 import * as bson from 'bson';
 import { createHash, randomUUID } from 'node:crypto';
+import { REPLICATION_WRITE_OPTIONS } from '../../MongoReplicationOptions.js';
 import { MongoWriteBatch } from '../../MongoWriteBatch.js';
 import { ObjectStorageDeletionMarker } from '../models.js';
 import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
@@ -8,6 +10,12 @@ import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
 import { ObjectStorage, ObjectStorageOperationOptions } from './ObjectStorage.js';
 
 export const OBJECT_STORAGE_UPLOAD_LEASE_MS = 60 * 60 * 1000;
+/**
+ * Leave time for the bounded publication transaction and clock skew before
+ * cleanup can select a marker. Publication must finish within this margin after
+ * its eligibility check; checked marker deletion alone cannot protect an object
+ * from a cleanup worker that already selected an expired marker.
+ */
 export const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS = 2 * 60 * 1000;
 export const OBJECT_STORAGE_REFERENCE_GRACE_MS = 15 * 60 * 1000;
 /**
@@ -23,6 +31,11 @@ export interface PreparedObjectStorageUpload {
   deleteAfter: Date;
 }
 
+export interface ObjectStorageUploadPayload {
+  upload: PreparedObjectStorageUpload;
+  payload: Uint8Array;
+}
+
 /**
  * Coordinates the MongoDB outbox used to make object-storage changes recoverable.
  *
@@ -67,18 +80,26 @@ export class ObjectStorageLifecycle {
    * Indicate that files will be uploaded to S3.
    *
    * These markers are expected to be removed using publishUploads(). If that does not happen within
-   * the grace period, the files will be deleted.
+   * the grace period, the files will be deleted. Registration must be majority
+   * acknowledged before any PUT starts: losing a marker on failover would lose
+   * the only cleanup obligation for an unpublished object. A failed/ambiguous
+   * insert must therefore prevent uploads, even if some markers were inserted.
    */
-  async prepareUploads(paths: string[], now = new Date()): Promise<PreparedObjectStorageUpload[]> {
+  async prepareUploads(
+    paths: string[],
+    now = new Date(),
+    session?: mongo.ClientSession
+  ): Promise<PreparedObjectStorageUpload[]> {
     const deleteAfter = new Date(now.getTime() + OBJECT_STORAGE_UPLOAD_LEASE_MS);
     const uploads = paths.map((path) => ({ markerId: new bson.ObjectId(), path, deleteAfter }));
-    if (uploads.length) {
+    if (uploads.length > 0) {
       await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(
         uploads.map((upload) => ({
           _id: upload.markerId,
           path: upload.path,
           delete_after: upload.deleteAfter
-        }))
+        })),
+        { ...REPLICATION_WRITE_OPTIONS, session }
       );
     }
     return uploads;
@@ -88,6 +109,57 @@ export class ObjectStorageLifecycle {
     return now.getTime() < upload.deleteAfter.getTime() - OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
   }
 
+  /**
+   * Upload a durably registered generation, joining every started PUT on failure.
+   * The shared adapter limits concurrency; its queue must observe the same signal
+   * as an active request so no queued PUT can start after the protection deadline.
+   *
+   * Failed/ambiguous PUTs retain their original markers. S3's bounded request
+   * timeouts are far shorter than the one-hour orphan grace, including when a
+   * response is lost or the process dies. Cancellation does not prove an object
+   * is absent, so never delete an object or its marker on upload failure.
+   */
+  async storeUploads(uploads: ObjectStorageUploadPayload[], options?: ObjectStorageOperationOptions): Promise<void> {
+    if (uploads.length === 0) {
+      return;
+    }
+    const deadline =
+      Math.min(...uploads.map(({ upload }) => upload.deleteAfter.getTime())) -
+      OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) {
+      throw new Error('Object storage upload lease expired before PUT');
+    }
+    const expiration = new AbortController();
+    const timer = setTimeout(
+      () => expiration.abort(new Error('Object storage upload lease expired during PUT')),
+      remaining
+    );
+    timer.unref();
+    const signal = options?.signal ? AbortSignal.any([options.signal, expiration.signal]) : expiration.signal;
+    let results: PromiseSettledResult<unknown>[];
+    try {
+      results = await Promise.allSettled(
+        uploads.map(async ({ upload, payload }) => {
+          signal.throwIfAborted();
+          // The timer handles time in the adapter's queue; this check also catches
+          // wall-clock changes before dispatch. Publication checks again later.
+          if (!this.canPublish(upload)) {
+            throw new Error(`Object storage upload lease expired for ${upload.path}`);
+          }
+          await this.bucketData.storeSerialized(upload.path, payload, { signal });
+        })
+      );
+    } finally {
+      clearTimeout(timer);
+    }
+    const failure = results.find((result) => result.status === 'rejected');
+    if (failure?.status === 'rejected') {
+      throw failure.reason;
+    }
+    signal.throwIfAborted();
+  }
+
   /**
    * Remove markers for files that have finished uploading, and are now referenced from other documents.
    *
@@ -107,7 +179,9 @@ export class ObjectStorageLifecycle {
       this.db.pendingObjectStorageDeletes(this.replicationStreamId),
       { _id: { $in: uploads.map((upload) => upload.markerId) } },
       (deletedCount) => {
-        if (deletedCount !== uploads.length) throw new Error('Missing object storage publication markers');
+        if (deletedCount !== uploads.length) {
+          throw new Error('Missing object storage publication markers');
+        }
       }
     );
   }

package/test/src/checkpoint_cleanup.test.ts

@@ -0,0 +1,75 @@
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { updateSyncRulesFromYaml } from '@powersync/service-core';
+import { getTestStorage, test_utils } from '@powersync/service-core-tests';
+import { setTimeout } from 'node:timers/promises';
+import { describe, expect, test } from 'vitest';
+import { env } from './env.js';
+import { TEST_STORAGE_VERSIONS } from './util.js';
+
+describe.each(TEST_STORAGE_VERSIONS)('checkpoint cleanup v%s', (storageVersion) => {
+  test('cancels an idle cursor without waiting for another checkpoint', async (context) => {
+    const monitored = mongoTestStorageFactoryGenerator({
+      url: env.MONGO_TEST_URL,
+      isCI: env.CI,
+      monitorCommands: true
+    });
+    await using factory = await monitored.factory();
+    const stream = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+config:
+  edition: 3
+streams:
+  test:
+    query: SELECT * FROM test
+`,
+        { storageVersion }
+      )
+    );
+    const bucketStorage = await getTestStorage(factory, stream);
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    await writer.markAllSnapshotDone('1/0');
+    await writer.keepalive('1/0');
+
+    const reading = Promise.withResolvers<void>();
+    const killed = Promise.withResolvers<void>();
+    const monitor = (event: mongo.CommandStartedEvent) => {
+      if (event.commandName === 'getMore' && event.command.collection === 'checkpoint_events') {
+        reading.resolve();
+      }
+    };
+    const finished = (event: mongo.CommandSucceededEvent) => {
+      if (event.commandName === 'killCursors') killed.resolve();
+    };
+    factory.db.client.on('commandStarted', monitor);
+    context.onTestFinished(() => {
+      factory.db.client.off('commandStarted', monitor);
+    });
+    factory.db.client.on('commandSucceeded', finished);
+    context.onTestFinished(() => {
+      factory.db.client.off('commandSucceeded', finished);
+    });
+    const controller = new AbortController();
+    context.onTestFinished(() => controller.abort());
+    const iterator = bucketStorage
+      .watchCheckpointChanges({ user_id: 'test', signal: controller.signal })
+      [Symbol.asyncIterator]();
+    await expect(iterator.next()).resolves.toMatchObject({ done: false, value: { base: { lsn: '1/0' } } });
+    const next = iterator.next().catch(() => {});
+    await reading.promise;
+
+    controller.abort();
+    const deadline = new AbortController();
+    try {
+      await Promise.race([
+        Promise.all([next, iterator.return?.(), killed.promise]),
+        setTimeout(2_000, undefined, { signal: deadline.signal }).then(() => {
+          throw new Error('Checkpoint cleanup waited for an idle getMore or a new checkpoint');
+        })
+      ]);
+    } finally {
+      deadline.abort();
+    }
+  });
+});