@powersync/service-module-mongodb-storage 0.10.3 → 0.11.0

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -2,11 +2,9 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import {
   BaseObserver,
-  ErrorCode,
   logger,
   ReplicationAbortedError,
-  ServiceAssertionError,
-  ServiceError
+  ServiceAssertionError
 } from '@powersync/lib-services-framework';
 import {
   BroadcastIterable,
@@ -16,6 +14,7 @@ import {
   GetCheckpointChangesOptions,
   InternalOpId,
   internalToExternalOpId,
+  maxLsn,
   ProtocolOpId,
   ReplicationCheckpoint,
   storage,
@@ -29,18 +28,12 @@ import { LRUCache } from 'lru-cache';
 import * as timers from 'timers/promises';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
-import {
-  BucketDataDocument,
-  BucketDataKey,
-  BucketStateDocument,
-  SourceKey,
-  SourceTableDocument,
-  SyncRuleCheckpointState
-} from './models.js';
+import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
-import { idPrefixFilter, mapOpEntry, readSingleBatch } from './util.js';
+import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from './util.js';
+import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 
 export class MongoSyncBucketStorage
   extends BaseObserver<storage.SyncRulesBucketStorageListener>
@@ -105,22 +98,44 @@ export class MongoSyncBucketStorage
   }
 
   async getCheckpoint(): Promise<storage.ReplicationCheckpoint> {
-    const doc = await this.db.sync_rules.findOne(
-      { _id: this.group_id },
-      {
-        projection: { last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
+    return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint();
+  }
+
+  async getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null> {
+    return await this.db.client.withSession({ snapshot: true }, async (session) => {
+      const doc = await this.db.sync_rules.findOne(
+        { _id: this.group_id },
+        {
+          session,
+          projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
+        }
+      );
+      if (!doc?.snapshot_done || !['ACTIVE', 'ERRORED'].includes(doc.state)) {
+        // Sync rules not active - return null
+        return null;
       }
-    );
-    if (!doc?.snapshot_done) {
-      return {
-        checkpoint: 0n,
-        lsn: null
-      };
-    }
-    return {
-      checkpoint: doc?.last_checkpoint ?? 0n,
-      lsn: doc?.last_checkpoint_lsn ?? null
-    };
+
+      // Specifically using operationTime instead of clusterTime
+      // There are 3 fields in the response:
+      // 1. operationTime, not exposed for snapshot sessions (used for causal consistency)
+      // 2. clusterTime (used for connection management)
+      // 3. atClusterTime, which is session.snapshotTime
+      // We use atClusterTime, to match the driver's internal snapshot handling.
+      // There are cases where clusterTime > operationTime and atClusterTime,
+      // which could cause snapshot queries using this as the snapshotTime to timeout.
+      // This was specifically observed on MongoDB 6.0 and 7.0.
+      const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined;
+      if (snapshotTime == null) {
+        throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
+      }
+      return new MongoReplicationCheckpoint(
+        this,
+        // null/0n is a valid checkpoint in some cases, for example if the initial snapshot was empty
+        doc.last_checkpoint ?? 0n,
+        doc.last_checkpoint_lsn ?? null,
+        snapshotTime
+      );
+    });
   }
 
   async startBatch(
@@ -131,7 +146,7 @@ export class MongoSyncBucketStorage
       {
         _id: this.group_id
       },
-      { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1, keepalive_op: 1 } }
+      { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1, keepalive_op: 1, snapshot_lsn: 1 } }
     );
     const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null;
 
@@ -142,6 +157,7 @@ export class MongoSyncBucketStorage
       groupId: this.group_id,
       slotName: this.slot_name,
       lastCheckpointLsn: checkpoint_lsn,
+      resumeFromLsn: maxLsn(checkpoint_lsn, doc?.snapshot_lsn),
       noCheckpointBeforeLsn: doc?.no_checkpoint_before ?? options.zeroLSN,
       keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
       storeCurrentData: options.storeCurrentData,
@@ -162,9 +178,9 @@ export class MongoSyncBucketStorage
   async resolveTable(options: storage.ResolveTableOptions): Promise<storage.ResolveTableResult> {
     const { group_id, connection_id, connection_tag, entity_descriptor } = options;
 
-    const { schema, name: table, objectId, replicationColumns } = entity_descriptor;
+    const { schema, name, objectId, replicaIdColumns } = entity_descriptor;
 
-    const columns = replicationColumns.map((column) => ({
+    const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
       name: column.name,
       type: column.type,
       type_oid: column.typeId
@@ -176,8 +192,8 @@ export class MongoSyncBucketStorage
         group_id: group_id,
         connection_id: connection_id,
         schema_name: schema,
-        table_name: table,
-        replica_id_columns2: columns
+        table_name: name,
+        replica_id_columns2: normalizedReplicaIdColumns
       };
       if (objectId != null) {
         filter.relation_id = objectId;
@@ -190,24 +206,24 @@ export class MongoSyncBucketStorage
           connection_id: connection_id,
           relation_id: objectId,
           schema_name: schema,
-          table_name: table,
+          table_name: name,
           replica_id_columns: null,
-          replica_id_columns2: columns,
+          replica_id_columns2: normalizedReplicaIdColumns,
           snapshot_done: false,
           snapshot_status: undefined
         };
 
         await col.insertOne(doc, { session });
       }
-      const sourceTable = new storage.SourceTable(
-        doc._id,
-        connection_tag,
-        objectId,
-        schema,
-        table,
-        replicationColumns,
-        doc.snapshot_done ?? true
-      );
+      const sourceTable = new storage.SourceTable({
+        id: doc._id,
+        connectionTag: connection_tag,
+        objectId: objectId,
+        schema: schema,
+        name: name,
+        replicaIdColumns: replicaIdColumns,
+        snapshotComplete: doc.snapshot_done ?? true
+      });
       sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable);
       sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable);
       sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable);
@@ -222,7 +238,7 @@ export class MongoSyncBucketStorage
 
       let dropTables: storage.SourceTable[] = [];
       // Detect tables that are either renamed, or have different replica_id_columns
-      let truncateFilter = [{ schema_name: schema, table_name: table }] as any[];
+      let truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
       if (objectId != null) {
         // Only detect renames if the source uses relation ids.
         truncateFilter.push({ relation_id: objectId });
@@ -240,15 +256,16 @@ export class MongoSyncBucketStorage
         .toArray();
       dropTables = truncate.map(
         (doc) =>
-          new storage.SourceTable(
-            doc._id,
-            connection_tag,
-            doc.relation_id,
-            doc.schema_name,
-            doc.table_name,
-            doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [],
-            doc.snapshot_done ?? true
-          )
+          new storage.SourceTable({
+            id: doc._id,
+            connectionTag: connection_tag,
+            objectId: doc.relation_id,
+            schema: doc.schema_name,
+            name: doc.table_name,
+            replicaIdColumns:
+              doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [],
+            snapshotComplete: doc.snapshot_done ?? true
+          })
       );
 
       result = {
@@ -259,38 +276,67 @@ export class MongoSyncBucketStorage
     return result!;
   }
 
-  async getParameterSets(checkpoint: utils.InternalOpId, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]> {
-    const lookupFilter = lookups.map((lookup) => {
-      return storage.serializeLookup(lookup);
-    });
-    const rows = await this.db.bucket_parameters
-      .aggregate([
-        {
-          $match: {
-            'key.g': this.group_id,
-            lookup: { $in: lookupFilter },
-            _id: { $lte: checkpoint }
-          }
-        },
-        {
-          $sort: {
-            _id: -1
-          }
-        },
-        {
-          $group: {
-            _id: { key: '$key', lookup: '$lookup' },
-            bucket_parameters: {
-              $first: '$bucket_parameters'
+  async getParameterSets(checkpoint: MongoReplicationCheckpoint, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]> {
+    return this.db.client.withSession({ snapshot: true }, async (session) => {
+      // Set the session's snapshot time to the checkpoint's snapshot time.
+      // An alternative would be to create the session when the checkpoint is created, but managing
+      // the session lifetime would become more complex.
+      // Starting and ending sessions are cheap (synchronous when no transactions are used),
+      // so this should be fine.
+      // This is a roundabout way of setting {readConcern: {atClusterTime: clusterTime}}, since
+      // that is not exposed directly by the driver.
+      // Future versions of the driver may change the snapshotTime behavior, so we need tests to
+      // validate that this works as expected. We test this in the compacting tests.
+      setSessionSnapshotTime(session, checkpoint.snapshotTime);
+      const lookupFilter = lookups.map((lookup) => {
+        return storage.serializeLookup(lookup);
+      });
+      // This query does not use indexes super efficiently, apart from the lookup filter.
+      // From some experimentation I could do individual lookups more efficient using an index
+      // on {'key.g': 1, lookup: 1, 'key.t': 1, 'key.k': 1, _id: -1},
+      // but could not do the same using $group.
+      // For now, just rely on compacting to remove extraneous data.
+      // For a description of the data format, see the `/docs/parameters-lookups.md` file.
+      const rows = await this.db.bucket_parameters
+        .aggregate(
+          [
+            {
+              $match: {
+                'key.g': this.group_id,
+                lookup: { $in: lookupFilter },
+                _id: { $lte: checkpoint.checkpoint }
+              }
+            },
+            {
+              $sort: {
+                _id: -1
+              }
+            },
+            {
+              $group: {
+                _id: { key: '$key', lookup: '$lookup' },
+                bucket_parameters: {
+                  $first: '$bucket_parameters'
+                }
+              }
             }
+          ],
+          {
+            session,
+            readConcern: 'snapshot',
+            // Limit the time for the operation to complete, to avoid getting connection timeouts
+            maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
           }
-        }
-      ])
-      .toArray();
-    const groupedParameters = rows.map((row) => {
-      return row.bucket_parameters;
+        )
+        .toArray()
+        .catch((e) => {
+          throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries');
+        });
+      const groupedParameters = rows.map((row) => {
+        return row.bucket_parameters;
+      });
+      return groupedParameters.flat();
     });
-    return groupedParameters.flat();
   }
 
   async *getBucketDataBatch(
@@ -348,7 +394,10 @@ export class MongoSyncBucketStorage
         // 1. We can calculate the document size accurately without serializing again.
         // 2. We can delay parsing the results until it's needed.
         // We manually use bson.deserialize below
-        raw: true
+        raw: true,
+
+        // Limit the time for the operation to complete, to avoid getting connection timeouts
+        maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
       }
     ) as unknown as mongo.FindCursor<Buffer>;
 
@@ -357,7 +406,9 @@ export class MongoSyncBucketStorage
     // to the lower of the batch count and size limits.
     // This is similar to using `singleBatch: true` in the find options, but allows
     // detecting "hasMore".
-    let { data, hasMore: batchHasMore } = await readSingleBatch(cursor);
+    let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => {
+      throw lib_mongo.mapQueryError(e, 'while reading bucket data');
+    });
     if (data.length == batchLimit) {
       // Limit reached - could have more data, despite the cursor being drained.
       batchHasMore = true;
@@ -486,9 +537,12 @@ export class MongoSyncBucketStorage
             }
           }
         ],
-        { session: undefined, readConcern: 'snapshot' }
+        { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS }
       )
-      .toArray();
+      .toArray()
+      .catch((e) => {
+        throw lib_mongo.mapQueryError(e, 'while reading checksums');
+      });
 
     return new Map<string, storage.PartialChecksum>(
       aggregate.map((doc) => {
@@ -567,7 +621,6 @@ export class MongoSyncBucketStorage
             `${this.slot_name} Cleared batch of data in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...`
           );
           await timers.setTimeout(lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5);
-          continue;
         } else {
           throw e;
         }
@@ -632,41 +685,6 @@ export class MongoSyncBucketStorage
     );
   }
 
-  async autoActivate(): Promise<void> {
-    await this.db.client.withSession(async (session) => {
-      await session.withTransaction(async () => {
-        const doc = await this.db.sync_rules.findOne({ _id: this.group_id }, { session });
-        if (doc && doc.state == 'PROCESSING') {
-          await this.db.sync_rules.updateOne(
-            {
-              _id: this.group_id
-            },
-            {
-              $set: {
-                state: storage.SyncRuleState.ACTIVE
-              }
-            },
-            { session }
-          );
-
-          await this.db.sync_rules.updateMany(
-            {
-              _id: { $ne: this.group_id },
-              state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
-            },
-            {
-              $set: {
-                state: storage.SyncRuleState.STOP
-              }
-            },
-            { session }
-          );
-          await this.db.notifyCheckpoint();
-        }
-      });
-    });
-  }
-
   async reportError(e: any): Promise<void> {
     const message = String(e.message ?? 'Replication failure');
     await this.db.sync_rules.updateOne(
@@ -683,14 +701,11 @@ export class MongoSyncBucketStorage
   }
 
   async compact(options?: storage.CompactOptions) {
-    return new MongoCompactor(this.db, this.group_id, options).compact();
-  }
-
-  private makeActiveCheckpoint(doc: SyncRuleCheckpointState | null) {
-    return {
-      checkpoint: doc?.last_checkpoint ?? 0n,
-      lsn: doc?.last_checkpoint_lsn ?? null
-    };
+    const checkpoint = await this.getCheckpointInternal();
+    await new MongoCompactor(this.db, this.group_id, options).compact();
+    if (checkpoint != null && options?.compactParameterData) {
+      await new MongoParameterCompactor(this.db, this.group_id, checkpoint.checkpoint, options).compact();
+    }
   }
 
   /**
@@ -712,33 +727,13 @@ export class MongoSyncBucketStorage
         break;
       }
 
-      const doc = await this.db.sync_rules.findOne(
-        {
-          _id: this.group_id,
-          state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
-        },
-        {
-          limit: 1,
-          projection: {
-            _id: 1,
-            state: 1,
-            last_checkpoint: 1,
-            last_checkpoint_lsn: 1
-          }
-        }
-      );
-
-      if (doc == null) {
-        // Sync rules not present or not active.
-        // Abort the connections - clients will have to retry later.
-        throw new ServiceError(ErrorCode.PSYNC_S2302, 'No active sync rules available');
-      } else if (doc.state != storage.SyncRuleState.ACTIVE && doc.state != storage.SyncRuleState.ERRORED) {
+      const op = await this.getCheckpointInternal();
+      if (op == null) {
         // Sync rules have changed - abort and restart.
         // We do a soft close of the stream here - no error
         break;
       }
 
-      const op = this.makeActiveCheckpoint(doc);
       // Check for LSN / checkpoint changes - ignore other metadata changes
       if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
         lastOp = op;
@@ -1005,3 +1000,25 @@ interface InternalCheckpointChanges extends CheckpointChanges {
   updatedWriteCheckpoints: Map<string, bigint>;
   invalidateWriteCheckpoints: boolean;
 }
+
+class MongoReplicationCheckpoint implements ReplicationCheckpoint {
+  constructor(
+    private storage: MongoSyncBucketStorage,
+    public readonly checkpoint: InternalOpId,
+    public readonly lsn: string | null,
+    public snapshotTime: mongo.Timestamp
+  ) {}
+
+  async getParameterSets(lookups: ParameterLookup[]): Promise<SqliteJsonRow[]> {
+    return this.storage.getParameterSets(this, lookups);
+  }
+}
+
+class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
+  readonly checkpoint: InternalOpId = 0n;
+  readonly lsn: string | null = null;
+
+  async getParameterSets(lookups: ParameterLookup[]): Promise<SqliteJsonRow[]> {
+    return [];
+  }
+}

package/src/storage/implementation/db.ts

@@ -1,6 +1,6 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
-import { storage } from '@powersync/service-core';
+import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
 
 import { MongoStorageConfig } from '../../types/types.js';
 import {
@@ -130,5 +130,11 @@ export class PowerSyncMongo {
 }
 
 export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) {
-  return new PowerSyncMongo(lib_mongo.createMongoClient(config, options), { database: config.database });
+  return new PowerSyncMongo(
+    lib_mongo.createMongoClient(config, {
+      powersyncVersion: POWERSYNC_VERSION,
+      ...options
+    }),
+    { database: config.database }
+  );
 }

package/src/storage/implementation/models.ts

@@ -118,9 +118,15 @@ export interface SyncRuleDocument {
   snapshot_done: boolean;
 
   /**
+   * This is now used for "resumeLsn".
+   *
    * If snapshot_done = false, this may be the lsn at which we started the snapshot.
    *
    * This can be used for resuming the snapshot after a restart.
+   *
+   * If snapshot_done is true, this is treated as the point to restart replication from.
+   *
+   * More specifically, we resume replication from max(snapshot_lsn, last_checkpoint_lsn).
    */
   snapshot_lsn: string | undefined;
 

package/src/storage/implementation/util.ts

@@ -7,6 +7,7 @@ import { storage, utils } from '@powersync/service-core';
 
 import { PowerSyncMongo } from './db.js';
 import { BucketDataDocument } from './models.js';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
   let filter = {
@@ -104,23 +105,28 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId):
   }
 }
 
-/**
- * Helper function for creating a MongoDB client from consumers of this package
- */
-export const createMongoClient = (url: string, options?: mongo.MongoClientOptions) => {
-  return new mongo.MongoClient(url, options);
-};
-
 /**
  * Helper for unit tests
  */
 export const connectMongoForTests = (url: string, isCI: boolean) => {
   // Short timeout for tests, to fail fast when the server is not available.
   // Slightly longer timeouts for CI, to avoid arbitrary test failures
-  const client = createMongoClient(url, {
+  const client = new mongo.MongoClient(url, {
     connectTimeoutMS: isCI ? 15_000 : 5_000,
     socketTimeoutMS: isCI ? 15_000 : 5_000,
     serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
   });
   return new PowerSyncMongo(client);
 };
+
+export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp) {
+  // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
+  if (!session.snapshotEnabled) {
+    throw new ServiceAssertionError(`Session must be a snapshot session`);
+  }
+  if ((session as any).snapshotTime == null) {
+    (session as any).snapshotTime = time;
+  } else {
+    throw new ServiceAssertionError(`Session snapshotTime is already set`);
+  }
+}

package/test/src/storage_compacting.test.ts

@@ -3,3 +3,5 @@ import { describe } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 describe('Mongo Sync Bucket Storage Compact', () => register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+describe('Mongo Sync Parameter Storage Compact', () =>
+  register.registerParameterCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY));