@powersync/service-module-mongodb-storage 0.18.1 → 0.18.2

package/src/storage/implementation/v1/MongoChecksumsV1.ts

@@ -13,15 +13,16 @@ import {
   checksumFromAggregate,
   DEFAULT_OPERATION_BATCH_LIMIT,
   FetchPartialBucketChecksumByBucket,
-  MongoChecksums
+  MongoChecksums,
+  MongoChecksumSessionContext
 } from '../MongoChecksums.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
-
 export class MongoChecksumsV1 extends MongoChecksums {
   declare protected readonly db: VersionedPowerSyncMongoV1;
 
   async computePartialChecksumsDirectByBucket(
-    batch: FetchPartialBucketChecksumByBucket[]
+    batch: FetchPartialBucketChecksumByBucket[],
+    context: MongoChecksumSessionContext
   ): Promise<PartialChecksumMap> {
     const collection = this.db.bucketDataV1;
     const createFilter = (request: FetchPartialBucketChecksumByBucket) => ({
@@ -95,8 +96,7 @@ export class MongoChecksumsV1 extends MongoChecksums {
             { $sort: { _id: 1 } }
           ],
           {
-            session: undefined,
-            readConcern: 'snapshot',
+            ...context.readOptions,
             maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS
           }
         )
@@ -164,7 +164,8 @@ export class MongoChecksumsV1 extends MongoChecksums {
   }
 
   protected async fetchPreStates(
-    batch: FetchPartialBucketChecksum[]
+    batch: FetchPartialBucketChecksum[],
+    context: MongoChecksumSessionContext
   ): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>> {
     const preFilters = batch
       .filter((request) => request.start == null)
@@ -182,9 +183,14 @@ export class MongoChecksumsV1 extends MongoChecksums {
     }
 
     const states = await this.db.bucketStateV1
-      .find({
-        $or: preFilters
-      })
+      .find(
+        {
+          $or: preFilters
+        },
+        {
+          ...context.readOptions
+        }
+      )
       .toArray();
 
     for (const state of states) {
@@ -202,7 +208,10 @@ export class MongoChecksumsV1 extends MongoChecksums {
     return preStates;
   }
 
-  protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
-    return this.computePartialChecksumsDirectByBucket(batch);
+  protected async computePartialChecksumsInternal(
+    batch: FetchPartialBucketChecksum[],
+    context: MongoChecksumSessionContext
+  ): Promise<PartialChecksumMap> {
+    return this.computePartialChecksumsDirectByBucket(batch, context);
   }
 }

package/src/storage/implementation/v1/MongoCompactorV1.ts

@@ -64,7 +64,8 @@ export class MongoCompactorV1 extends MongoCompactor {
       buckets.map(({ bucket }) => ({
         bucket,
         end: this.maxOpId
-      }))
+      })),
+      { readOptions: { readConcern: 'snapshot' } }
     );
   }
 

package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts

@@ -41,6 +41,7 @@ import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 export interface MongoSyncBucketStorageContextV1 {
   db: VersionedPowerSyncMongoV1;
   replicationStreamId: number;
+  readPreference?: mongo.ReadPreference;
 }
 
 export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
@@ -172,6 +173,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
   protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums {
     return new MongoChecksumsV1(this.db, this.replicationStreamId, {
       ...options.checksumOptions,
+      checksumCacheTtlMs: options.checksumCacheTtlMs,
       storageConfig: options?.storageConfig
     });
   }
@@ -190,7 +192,8 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
   protected get versionContext(): MongoSyncBucketStorageContextV1 {
     return {
       db: this.db,
-      replicationStreamId: this.replicationStreamId
+      replicationStreamId: this.replicationStreamId,
+      readPreference: this.readPreference
     };
   }
 
@@ -203,7 +206,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
   }
 
   protected getBucketDataBatchImpl(
-    checkpoint: utils.InternalOpId,
+    checkpoint: MongoSyncBucketStorageCheckpoint,
     dataBuckets: storage.BucketDataRequest[],
     options?: storage.BucketDataBatchOptions
   ): AsyncIterable<storage.SyncBucketDataChunk> {
@@ -369,20 +372,32 @@ export async function getParameterSetsV1(
 
 export async function* getBucketDataBatchV1(
   ctx: MongoSyncBucketStorageContextV1,
-  checkpoint: utils.InternalOpId,
+  checkpoint: MongoSyncBucketStorageCheckpoint,
   dataBuckets: storage.BucketDataRequest[],
   options?: storage.BucketDataBatchOptions
 ): AsyncIterable<storage.SyncBucketDataChunk> {
   if (dataBuckets.length == 0) {
     return;
   }
+  const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
+  const readConcern = ctx.readPreference == null ? undefined : 'majority';
+  const session =
+    readPreference == null || checkpoint.snapshotTime == null
+      ? undefined
+      : ctx.db.client.startSession({ causalConsistency: true });
+  await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
+
+  if (session != null) {
+    session.advanceOperationTime(checkpoint.snapshotTime);
+  }
+
   let filters: mongo.Filter<BucketDataDocumentV1>[] = [];
   const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
 
-  if (checkpoint == null) {
+  if (checkpoint.checkpoint == null) {
     throw new Error('checkpoint is null');
   }
-  const end = checkpoint;
+  const end = checkpoint.checkpoint;
   for (let { bucket: name, start } of dataBuckets) {
     filters.push({
       _id: {
@@ -410,7 +425,9 @@ export async function* getBucketDataBatchV1(
       $or: filters
     },
     {
-      session: undefined,
+      session,
+      readPreference,
+      readConcern,
       sort: { _id: 1 },
       limit: batchLimit,
       batchSize: batchLimit + 1,

package/src/storage/implementation/v3/MongoBucketBatchV3.ts

@@ -243,6 +243,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
 
     await this.flush(options);
 
+    using _ = this.tracer.span('storage', 'commit');
+
     const now = new Date();
 
     await this.db.write_checkpoints.updateMany(
@@ -418,6 +420,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
   }
 
   async setResumeLsn(lsn: string): Promise<void> {
+    using _ = this.tracer.span('storage', 'set_resume_lsn');
     await this.db.sync_rules.updateOne(
       {
         _id: this.replicationStreamId
@@ -427,7 +430,13 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
           resume_lsn: lsn
         }
       },
-      { session: this.session }
+      {
+        session: this.session,
+        // Losing occasional resume LSN is fine. That may mean reprocessing
+        // some source changes in some edge cases, which is not an issue since
+        // changes are processed in an idempotent way.
+        writeConcern: { w: 1 }
+      }
     );
   }
 
@@ -438,7 +447,12 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
 
     const session = this.session;
     let activated = false;
+    let needsFutureActivationCheck = true;
     await session.withTransaction(async () => {
+      // Reset on transaction retries.
+      needsFutureActivationCheck = true;
+      activated = false;
+
       const doc = await this.db.sync_rules.findOne(
         {
           _id: this.replicationStreamId,
@@ -518,13 +532,17 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
         );
         activated = true;
       } else if (doc.state != storage.SyncRuleState.PROCESSING && doc.state != storage.SyncRuleState.ACTIVE) {
-        this.needsActivationV3 = false;
+        needsFutureActivationCheck = false;
+      } else if (doc.state == storage.SyncRuleState.ACTIVE && processingStates.length == 0) {
+        needsFutureActivationCheck = false;
       }
     });
     if (activated) {
       this.logger.info(`Activated new replication stream at ${lsn}`);
       await this.db.notifyCheckpoint();
       this.needsActivationV3 = false;
+    } else if (!needsFutureActivationCheck) {
+      this.needsActivationV3 = false;
     }
   }
 

package/src/storage/implementation/v3/MongoChecksumsV3.ts

@@ -15,7 +15,8 @@ import {
   FetchPartialBucketChecksumByBucket,
   FetchPartialBucketChecksumByDefinition,
   MongoChecksumOptions,
-  MongoChecksums
+  MongoChecksums,
+  MongoChecksumSessionContext
 } from '../MongoChecksums.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 import { BucketDataDocumentV3 } from './models.js';
@@ -66,7 +67,8 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
     for (const [definitionId, requests] of requestsByDefinition.entries()) {
       const groupResults = await this.computeChecksumsByDefinition(
         requests,
-        this.db.bucketData(this.group_id, definitionId)
+        this.db.bucketData(this.group_id, definitionId),
+        { readOptions: { readConcern: 'snapshot' } }
       );
       for (const checksum of groupResults.values()) {
         results.set(checksum.bucket, checksum);
@@ -79,7 +81,8 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
   }
 
   protected async fetchPreStates(
-    batch: FetchPartialBucketChecksum[]
+    batch: FetchPartialBucketChecksum[],
+    context: MongoChecksumSessionContext
   ): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>> {
     const normalizedBatch = batch.map((request) => ({
       bucket: request.bucket,
@@ -105,9 +108,14 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
 
     const states = await this.db
       .bucketState(this.group_id)
-      .find({
-        $or: preFilters
-      })
+      .find(
+        {
+          $or: preFilters
+        },
+        {
+          ...context.readOptions
+        }
+      )
       .toArray();
 
     for (const state of states) {
@@ -125,7 +133,10 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
     return preStates;
   }
 
-  protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
+  protected async computePartialChecksumsInternal(
+    batch: FetchPartialBucketChecksum[],
+    context: MongoChecksumSessionContext
+  ): Promise<PartialChecksumMap> {
     const normalized = batch.map((request) => ({
       bucket: request.bucket,
       definitionId: this.syncConfigMapping().bucketSourceId(request.source),
@@ -145,7 +156,8 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
     for (const [definitionId, requests] of requestsByDefinition.entries()) {
       const groupResults = await this.computeChecksumsByDefinition(
         requests,
-        this.db.bucketData(this.group_id, definitionId)
+        this.db.bucketData(this.group_id, definitionId),
+        context
       );
       for (const checksum of groupResults.values()) {
         results.set(checksum.bucket, checksum);
@@ -159,7 +171,8 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
 
   private async computeChecksumsByDefinition(
     batch: FetchPartialBucketChecksumByBucket[],
-    collection: lib_mongo.mongo.Collection<BucketDataDocumentV3>
+    collection: lib_mongo.mongo.Collection<BucketDataDocumentV3>,
+    context: MongoChecksumSessionContext
   ): Promise<PartialChecksumMap> {
     const requests = new Map<string, FetchPartialBucketChecksumByBucket>();
     for (let request of batch) {
@@ -169,8 +182,7 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
     const pipeline = this.buildPartialChecksumPipeline(requests);
     const aggregate = await collection
       .aggregate(pipeline, {
-        session: undefined,
-        readConcern: 'snapshot',
+        ...context.readOptions,
         maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS
       })
       .toArray()

package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts

@@ -42,6 +42,7 @@ import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 export interface MongoSyncBucketStorageContextV3 {
   db: VersionedPowerSyncMongoV3;
   replicationStreamId: number;
+  readPreference?: mongo.ReadPreference;
   /**
    * Persisted mapping of the single sync config that read operations are served from.
    *
@@ -171,6 +172,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
   protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums {
     return new MongoChecksumsV3(this.db, this.replicationStreamId, {
       ...options.checksumOptions,
+      checksumCacheTtlMs: options.checksumCacheTtlMs,
       storageConfig: options?.storageConfig,
       syncConfigMapping: () => this.singleSyncConfigMapping()
     });
@@ -336,6 +338,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     return {
       db: this.db,
       replicationStreamId: this.replicationStreamId,
+      readPreference: this.readPreference,
       get mapping() {
         return self.singleSyncConfigMapping();
       }
@@ -351,7 +354,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
   }
 
   protected getBucketDataBatchImpl(
-    checkpoint: utils.InternalOpId,
+    checkpoint: MongoSyncBucketStorageCheckpoint,
     dataBuckets: storage.BucketDataRequest[],
     options?: storage.BucketDataBatchOptions
   ): AsyncIterable<storage.SyncBucketDataChunk> {
@@ -528,7 +531,7 @@ export async function getParameterSetsV3(
 
 export async function* getBucketDataBatchV3(
   ctx: MongoSyncBucketStorageContextV3,
-  checkpoint: utils.InternalOpId,
+  checkpoint: MongoSyncBucketStorageCheckpoint,
   dataBuckets: storage.BucketDataRequest[],
   options?: storage.BucketDataBatchOptions
 ): AsyncIterable<storage.SyncBucketDataChunk> {
@@ -536,13 +539,25 @@ export async function* getBucketDataBatchV3(
     return;
   }
 
-  if (checkpoint == null) {
+  if (checkpoint.checkpoint == null) {
     throw new Error('checkpoint is null');
   }
 
+  const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
+  const readConcern = ctx.readPreference == null ? undefined : 'majority';
+  const session =
+    readPreference == null || checkpoint.snapshotTime == null
+      ? undefined
+      : ctx.db.client.startSession({ causalConsistency: true });
+  await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
+
+  if (session != null) {
+    session.advanceOperationTime(checkpoint.snapshotTime);
+  }
+
   const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
   const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
-  const end = checkpoint;
+  const end = checkpoint.checkpoint;
   let remainingLimit = batchLimit;
 
   const requestsByDefinition = new Map<string, storage.BucketDataRequest[]>();
@@ -580,7 +595,9 @@ export async function* getBucketDataBatchV3(
     // raw: true returns Buffers, but the driver typing doesn't reflect that
     // without an explicit cast to FindCursor<Buffer>.
     const cursor = collection.find(filter, {
-      session: undefined,
+      session,
+      readPreference,
+      readConcern,
       sort: { _id: 1 },
       raw: true,
       maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS,

package/src/types/types.ts

@@ -2,9 +2,25 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
 
+export const MongoStorageReadPreference = t
+  .literal('primary')
+  .or(t.literal('primaryPreferred'))
+  .or(t.literal('secondary'))
+  .or(t.literal('secondaryPreferred'))
+  .or(t.literal('nearest'));
+
+export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
+
 export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(
   t.object({
-    // Add any mongo specific storage settings here in future
+    /**
+     * Read preference for bulk checksum and bucket data reads.
+     *
+     * If unset, MongoDB driver defaults are used for backwards compatibility.
+     *
+     * This is an experimental option, and may be removed in a future release.
+     */
+    bulk_read_preference: MongoStorageReadPreference.optional()
   })
 );
 

package/test/src/storage_compacting.test.ts

@@ -95,7 +95,7 @@ bucket_definitions:
       const users = ['u1', 'u2'];
       const userRequests = users.map((user) => bucketRequest(syncRules, `by_user["${user}"]`));
       const [u1Request, u2Request] = userRequests;
-      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), userRequests);
       expect(checksumAfter.get(u1Request.bucket)).toEqual({
         bucket: u1Request.bucket,
         checksum: -659469718,
@@ -151,7 +151,7 @@ bucket_definitions:
       const users = ['u1', 'u2'];
       const userRequests = users.map((user) => bucketRequest(syncRulesContent, `by_user2["${user}"]`));
       const [u1Request, u2Request] = userRequests;
-      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), userRequests);
       expect(checksumAfter.get(u1Request.bucket)).toEqual({
         bucket: u1Request.bucket,
         checksum: -659469718,
@@ -869,7 +869,7 @@ bucket_definitions:
       } as any
     };
 
-    const result = await bucketStorage.getChecksums(60n, [request]);
+    const result = await bucketStorage.getChecksums(test_utils.testCheckpoint(60n), [request]);
     const checksumResult = result.get(BUCKET)!;
 
     // The total checksum should be: compacted (ops 10,20,30) + partial (ops 40,50,60)
@@ -922,7 +922,7 @@ bucket_definitions:
       } as any
     };
 
-    const result = await bucketStorage.getChecksums(45n, [request]);
+    const result = await bucketStorage.getChecksums(test_utils.testCheckpoint(45n), [request]);
     const checksumResult = result.get(BUCKET)!;
 
     // If createBucketFilter's _id.o <= 45 excludes this document,

package/test/src/storage_sync.test.ts

@@ -192,7 +192,11 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
 
     const options: storage.BucketDataBatchOptions = {};
     const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)], options)
+      bucketStorage.getBucketDataBatch(
+        test_utils.testCheckpoint(checkpoint),
+        [bucketRequest(syncRulesContent, 'global[]', 0n)],
+        options
+      )
     );
     expect(test_utils.getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -206,7 +210,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
 
     const batch2 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
-        checkpoint,
+        test_utils.testCheckpoint(checkpoint),
         [bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)],
         options
       )
@@ -222,7 +226,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
 
     const batch3 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
-        checkpoint,
+        test_utils.testCheckpoint(checkpoint),
         [bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)],
         options
       )
@@ -1743,7 +1747,9 @@ describe('sync - mongodb', () => {
         async function getFilteredOps(start: number, checkpoint: number): Promise<bigint[]> {
           const { syncRules, bucketStorage } = await setupFilteringTest();
           const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', BigInt(start));
-          const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(BigInt(checkpoint), [request]));
+          const batch = await test_utils.fromAsync(
+            bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(BigInt(checkpoint)), [request])
+          );
           const ops = batch.flatMap((b) => b.chunkData.data.map((d) => BigInt(d.op_id)));
           return ops;
         }
@@ -1888,7 +1894,9 @@ describe('sync - mongodb', () => {
             const roundRequests = requests
               .filter((request) => pending.has(request.bucket))
               .map((request) => ({ ...request, start: positions.get(request.bucket)! }));
-            const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(end, roundRequests));
+            const batch = await test_utils.fromAsync(
+              bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(end), roundRequests)
+            );
             let anyHasMore = false;
             for (const { chunkData } of batch) {
               positions.set(chunkData.bucket, BigInt(chunkData.next_after));

package/test/src/storeCurrentData.test.ts

@@ -131,7 +131,9 @@ function registerStoreCurrentDataTests(storageVersion: number) {
     const checkpoint = flushResult!.flushed_op;
 
     const batch = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [
+        bucketRequest(syncRulesContent, 'global[]', 0n)
+      ])
     );
     expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
 
@@ -161,7 +163,9 @@ function registerStoreCurrentDataTests(storageVersion: number) {
     const checkpoint = flushResult!.flushed_op;
 
     const batch = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [
+        bucketRequest(syncRulesContent, 'global[]', 0n)
+      ])
     );
     expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
 
@@ -200,7 +204,9 @@ function registerStoreCurrentDataTests(storageVersion: number) {
     const checkpoint = flushResult!.flushed_op;
 
     const batch = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [
+        bucketRequest(syncRulesContent, 'global[]', 0n)
+      ])
     );
     const data = test_utils.getBatchData(batch);
     expect(data.length).toBeGreaterThan(0);