@powersync/service-module-mongodb-storage 0.5.1 → 0.6.0

package/src/storage/implementation/MongoBucketBatch.ts

@@ -3,15 +3,15 @@ import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-
 import * as bson from 'bson';
 
 import {
+  BaseObserver,
   container,
-  DisposableObserver,
   ErrorCode,
   errors,
   logger,
   ReplicationAssertionError,
   ServiceError
 } from '@powersync/lib-services-framework';
-import { SaveOperationTag, storage, utils } from '@powersync/service-core';
+import { deserializeBson, SaveOperationTag, storage, utils } from '@powersync/service-core';
 import * as timers from 'node:timers/promises';
 import { PowerSyncMongo } from './db.js';
 import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
@@ -49,7 +49,7 @@ export interface MongoBucketBatchOptions {
 }
 
 export class MongoBucketBatch
-  extends DisposableObserver<storage.BucketBatchStorageListener>
+  extends BaseObserver<storage.BucketBatchStorageListener>
   implements storage.BucketStorageBatch
 {
   private readonly client: mongo.MongoClient;
@@ -322,10 +322,7 @@ export class MongoBucketBatch
         existing_buckets = result.buckets;
         existing_lookups = result.lookups;
         if (this.storeCurrentData) {
-          const data = bson.deserialize(
-            (result.data as mongo.Binary).buffer,
-            storage.BSON_DESERIALIZE_OPTIONS
-          ) as SqliteRow;
+          const data = deserializeBson((result.data as mongo.Binary).buffer) as SqliteRow;
           after = storage.mergeToast(after!, data);
         }
       }
@@ -610,7 +607,7 @@ export class MongoBucketBatch
 
   async [Symbol.asyncDispose]() {
     await this.session.endSession();
-    super[Symbol.dispose]();
+    super.clearListeners();
   }
 
   private lastWaitingLogThottled = 0;

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -1,20 +1,36 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
-import { DisposableObserver, logger, ServiceAssertionError } from '@powersync/lib-services-framework';
-import { storage, utils } from '@powersync/service-core';
+import {
+  BaseObserver,
+  ErrorCode,
+  logger,
+  ServiceAssertionError,
+  ServiceError
+} from '@powersync/lib-services-framework';
+import {
+  BroadcastIterable,
+  CHECKPOINT_INVALIDATE_ALL,
+  CheckpointChanges,
+  GetCheckpointChangesOptions,
+  ReplicationCheckpoint,
+  storage,
+  utils,
+  WatchWriteCheckpointOptions
+} from '@powersync/service-core';
 import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
+import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
 import * as timers from 'timers/promises';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument, BucketDataKey, SourceKey } from './models.js';
+import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleCheckpointState, SyncRuleDocument } from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 import { idPrefixFilter, mapOpEntry, readSingleBatch } from './util.js';
 
 export class MongoSyncBucketStorage
-  extends DisposableObserver<storage.SyncRulesBucketStorageListener>
+  extends BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
 {
   private readonly db: PowerSyncMongo;
@@ -324,7 +340,7 @@ export class MongoSyncBucketStorage
 
     // Ordered by _id, meaning buckets are grouped together
     for (let rawData of data) {
-      const row = bson.deserialize(rawData, storage.BSON_DESERIALIZE_OPTIONS) as BucketDataDocument;
+      const row = bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocument;
       const bucket = row._id.b;
 
       if (currentBatch == null || currentBatch.bucket != bucket || batchSize >= sizeLimit) {
@@ -608,4 +624,232 @@ export class MongoSyncBucketStorage
   async compact(options?: storage.CompactOptions) {
     return new MongoCompactor(this.db, this.group_id, options).compact();
   }
+
+  private makeActiveCheckpoint(doc: SyncRuleCheckpointState | null) {
+    return {
+      checkpoint: utils.timestampToOpId(doc?.last_checkpoint ?? 0n),
+      lsn: doc?.last_checkpoint_lsn ?? null
+    };
+  }
+
+  /**
+   * Instance-wide watch on the latest available checkpoint (op_id + lsn).
+   */
+  private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ReplicationCheckpoint> {
+    // Use this form instead of (doc: SyncRuleCheckpointState | null = null),
+    // otherwise we get weird "doc: never" issues.
+    let doc = null as SyncRuleCheckpointState | null;
+    let clusterTime = null as mongo.Timestamp | null;
+    const syncRulesId = this.group_id;
+
+    await this.db.client.withSession(async (session) => {
+      doc = await this.db.sync_rules.findOne(
+        {
+          _id: syncRulesId,
+          state: storage.SyncRuleState.ACTIVE
+        },
+        {
+          session,
+          sort: { _id: -1 },
+          limit: 1,
+          projection: {
+            _id: 1,
+            state: 1,
+            last_checkpoint: 1,
+            last_checkpoint_lsn: 1
+          }
+        }
+      );
+      const time = session.clusterTime?.clusterTime ?? null;
+      clusterTime = time;
+    });
+    if (clusterTime == null) {
+      throw new ServiceError(ErrorCode.PSYNC_S2401, 'Could not get clusterTime');
+    }
+
+    if (signal.aborted) {
+      return;
+    }
+
+    if (doc == null) {
+      // Sync rules not present or not active.
+      // Abort the connections - clients will have to retry later.
+      // Should this error instead?
+      return;
+    }
+
+    yield this.makeActiveCheckpoint(doc);
+
+    // We only watch changes to the active sync rules.
+    // If it changes to inactive, we abort and restart with the new sync rules.
+
+    const pipeline = this.getChangeStreamPipeline();
+
+    const stream = this.db.sync_rules.watch(pipeline, {
+      // Start at the cluster time where we got the initial doc, to make sure
+      // we don't skip any updates.
+      // This may result in the first operation being a duplicate, but we filter
+      // it out anyway.
+      startAtOperationTime: clusterTime
+    });
+
+    signal.addEventListener(
+      'abort',
+      () => {
+        stream.close();
+      },
+      { once: true }
+    );
+
+    let lastOp: storage.ReplicationCheckpoint | null = null;
+    let lastDoc: SyncRuleCheckpointState | null = doc;
+
+    for await (const update of stream.stream()) {
+      if (signal.aborted) {
+        break;
+      }
+      if (update.operationType != 'insert' && update.operationType != 'update' && update.operationType != 'replace') {
+        continue;
+      }
+
+      const doc = await this.getOperationDoc(lastDoc, update as lib_mongo.mongo.ChangeStreamDocument<SyncRuleDocument>);
+      if (doc == null) {
+        // Irrelevant update
+        continue;
+      }
+      if (doc.state != storage.SyncRuleState.ACTIVE) {
+        // Sync rules have changed - abort and restart.
+        // Should this error instead?
+        break;
+      }
+
+      lastDoc = doc;
+
+      const op = this.makeActiveCheckpoint(doc);
+      // Check for LSN / checkpoint changes - ignore other metadata changes
+      if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
+        lastOp = op;
+        yield op;
+      }
+    }
+  }
+
+  // Nothing is done here until a subscriber starts to iterate
+  private readonly sharedIter = new BroadcastIterable((signal) => {
+    return this.watchActiveCheckpoint(signal);
+  });
+
+  /**
+   * User-specific watch on the latest checkpoint and/or write checkpoint.
+   */
+  async *watchWriteCheckpoint(options: WatchWriteCheckpointOptions): AsyncIterable<storage.StorageCheckpointUpdate> {
+    const { user_id, signal } = options;
+    let lastCheckpoint: utils.OpId | null = null;
+    let lastWriteCheckpoint: bigint | null = null;
+
+    const iter = wrapWithAbort(this.sharedIter, signal);
+    for await (const event of iter) {
+      const { checkpoint, lsn } = event;
+
+      // lsn changes are not important by itself.
+      // What is important is:
+      // 1. checkpoint (op_id) changes.
+      // 2. write checkpoint changes for the specific user
+
+      const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
+
+      const currentWriteCheckpoint = await this.lastWriteCheckpoint({
+        user_id,
+        heads: {
+          ...lsnFilters
+        }
+      });
+
+      if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
+        // No change - wait for next one
+        // In some cases, many LSNs may be produced in a short time.
+        // Add a delay to throttle the write checkpoint lookup a bit.
+        await timers.setTimeout(20 + 10 * Math.random());
+        continue;
+      }
+
+      const updates: CheckpointChanges =
+        lastCheckpoint == null
+          ? {
+              invalidateDataBuckets: true,
+              invalidateParameterBuckets: true,
+              updatedDataBuckets: [],
+              updatedParameterBucketDefinitions: []
+            }
+          : await this.getCheckpointChanges({
+              lastCheckpoint: lastCheckpoint,
+              nextCheckpoint: checkpoint
+            });
+
+      lastWriteCheckpoint = currentWriteCheckpoint;
+      lastCheckpoint = checkpoint;
+
+      yield {
+        base: event,
+        writeCheckpoint: currentWriteCheckpoint,
+        update: updates
+      };
+    }
+  }
+
+  private async getOperationDoc(
+    lastDoc: SyncRuleCheckpointState,
+    update: lib_mongo.mongo.ChangeStreamDocument<SyncRuleDocument>
+  ): Promise<SyncRuleCheckpointState | null> {
+    if (update.operationType == 'insert' || update.operationType == 'replace') {
+      return update.fullDocument;
+    } else if (update.operationType == 'update') {
+      const updatedFields = update.updateDescription.updatedFields ?? {};
+      if (lastDoc._id != update.documentKey._id) {
+        throw new ServiceAssertionError(`Sync rules id mismatch: ${lastDoc._id} != ${update.documentKey._id}`);
+      }
+
+      const mergedDoc: SyncRuleCheckpointState = {
+        _id: lastDoc._id,
+        last_checkpoint: updatedFields.last_checkpoint ?? lastDoc.last_checkpoint,
+        last_checkpoint_lsn: updatedFields.last_checkpoint_lsn ?? lastDoc.last_checkpoint_lsn,
+        state: updatedFields.state ?? lastDoc.state
+      };
+
+      return mergedDoc;
+    } else {
+      // Unknown event type
+      return null;
+    }
+  }
+
+  private getChangeStreamPipeline() {
+    const syncRulesId = this.group_id;
+    const pipeline: mongo.Document[] = [
+      {
+        $match: {
+          'documentKey._id': syncRulesId,
+          operationType: { $in: ['insert', 'update', 'replace'] }
+        }
+      },
+      {
+        $project: {
+          operationType: 1,
+          'documentKey._id': 1,
+          'updateDescription.updatedFields.state': 1,
+          'updateDescription.updatedFields.last_checkpoint': 1,
+          'updateDescription.updatedFields.last_checkpoint_lsn': 1,
+          'fullDocument._id': 1,
+          'fullDocument.state': 1,
+          'fullDocument.last_checkpoint': 1,
+          'fullDocument.last_checkpoint_lsn': 1
+        }
+      }
+    ];
+    return pipeline;
+  }
+
+  async getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<CheckpointChanges> {
+    return CHECKPOINT_INVALIDATE_ALL;
+  }
 }

package/src/storage/implementation/db.ts

@@ -41,7 +41,7 @@ export class PowerSyncMongo {
     this.client = client;
 
     const db = client.db(options?.database, {
-      ...storage.BSON_DESERIALIZE_OPTIONS
+      ...storage.BSON_DESERIALIZE_INTERNAL_OPTIONS
     });
     this.db = db;
 

package/src/storage/implementation/models.ts

@@ -141,6 +141,11 @@ export interface SyncRuleDocument {
   content: string;
 }
 
+export type SyncRuleCheckpointState = Pick<
+  SyncRuleDocument,
+  'last_checkpoint' | 'last_checkpoint_lsn' | '_id' | 'state'
+>;
+
 export interface CustomWriteCheckpointDocument {
   _id: bson.ObjectId;
   user_id: string;

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -9,6 +9,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
           "bucket": "mybucket[]",
           "checksum": -93886621,
           "count": 2,
+          "priority": 3,
         },
       ],
       "last_op_id": "2",
@@ -44,6 +45,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
           "bucket": "mybucket[]",
           "checksum": 499012468,
           "count": 4,
+          "priority": 3,
         },
       ],
       "write_checkpoint": undefined,
@@ -102,6 +104,7 @@ exports[`sync - mongodb > expiring token 1`] = `
           "bucket": "mybucket[]",
           "checksum": 0,
           "count": 0,
+          "priority": 3,
         },
       ],
       "last_op_id": "0",
@@ -124,6 +127,80 @@ exports[`sync - mongodb > expiring token 2`] = `
 ]
 `;
 
+exports[`sync - mongodb > sync buckets in order 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "b0[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 2,
+        },
+        {
+          "bucket": "b1[]",
+          "checksum": -1382098757,
+          "count": 1,
+          "priority": 1,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "b1[]",
+      "data": [
+        {
+          "checksum": 2912868539n,
+          "data": "{"id":"earlier","description":"Test 2"}",
+          "object_id": "earlier",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0dfe86bd-d15b-5fd0-9c7b-a31693030ee0",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 1,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "b0[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > sync global data 1`] = `
 [
   {
@@ -133,6 +210,7 @@ exports[`sync - mongodb > sync global data 1`] = `
           "bucket": "mybucket[]",
           "checksum": -93886621,
           "count": 2,
+          "priority": 3,
         },
       ],
       "last_op_id": "2",
@@ -184,6 +262,7 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
           "bucket": "mybucket[]",
           "checksum": -852817836,
           "count": 1,
+          "priority": 3,
         },
       ],
       "last_op_id": "1",
@@ -231,6 +310,7 @@ exports[`sync - mongodb > sync updates to global data 1`] = `
           "bucket": "mybucket[]",
           "checksum": 0,
           "count": 0,
+          "priority": 3,
         },
       ],
       "last_op_id": "0",
@@ -256,6 +336,7 @@ exports[`sync - mongodb > sync updates to global data 2`] = `
           "bucket": "mybucket[]",
           "checksum": 920318466,
           "count": 1,
+          "priority": 3,
         },
       ],
       "write_checkpoint": undefined,
@@ -299,6 +380,7 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
           "bucket": "mybucket[]",
           "checksum": -93886621,
           "count": 2,
+          "priority": 3,
         },
       ],
       "write_checkpoint": undefined,

package/test/src/storage_sync.test.ts

@@ -19,7 +19,7 @@ describe('sync - mongodb', () => {
           - SELECT id, description FROM "%"
     `
     );
-    using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
     const bucketStorage = factory.getInstance(sync_rules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {