@powersync/service-module-mongodb-storage 0.9.5 → 0.10.1

package/dist/storage/implementation/db.js

@@ -12,6 +12,7 @@ export class PowerSyncMongo {
     instance;
     locks;
     bucket_state;
+    checkpoint_events;
     client;
     db;
     constructor(client, options) {
@@ -31,6 +32,7 @@ export class PowerSyncMongo {
         this.instance = db.collection('instance');
         this.locks = this.db.collection('locks');
         this.bucket_state = this.db.collection('bucket_state');
+        this.checkpoint_events = this.db.collection('checkpoint_events');
     }
     /**
      * Clear all collections.
@@ -56,6 +58,43 @@ export class PowerSyncMongo {
     async drop() {
         await this.db.dropDatabase();
     }
+    /**
+     * Call this after every checkpoint or sync rules status update. Rather call too often than too rarely.
+     *
+     * This is used in a similar way to the Postgres NOTIFY functionality.
+     */
+    async notifyCheckpoint() {
+        await this.checkpoint_events.insertOne({}, { forceServerObjectId: true });
+    }
+    /**
+     * Only use in migrations and tests.
+     */
+    async createCheckpointEventsCollection() {
+        // We cover the case where the replication process was started before running this migration.
+        const existingCollections = await this.db
+            .listCollections({ name: 'checkpoint_events' }, { nameOnly: false })
+            .toArray();
+        const collection = existingCollections[0];
+        if (collection != null) {
+            if (!collection.options?.capped) {
+                // Collection was auto-created but not capped, so we need to drop it
+                await this.db.dropCollection('checkpoint_events');
+            }
+            else {
+                // Collection previously created somehow - ignore
+                return;
+            }
+        }
+        await this.db.createCollection('checkpoint_events', {
+            capped: true,
+            // We want a small size, since opening a tailable cursor scans this entire collection.
+            // On the other hand, if we fill this up faster than a process can read it, it will
+            // invalidate the cursor. We do handle cursor invalidation events, but don't want
+            // that to happen too often.
+            size: 50 * 1024, // size in bytes
+            max: 50 // max number of documents
+        });
+    }
 }
 export function createPowerSyncMongo(config, options) {
     return new PowerSyncMongo(lib_mongo.createMongoClient(config, options), { database: config.database });

package/dist/storage/implementation/db.js.map

@@ -1 +1 @@
-{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAuBlD,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAAwC;IAEpD,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AACzG,CAAC"}
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwBlD,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAAwC;IACpD,iBAAiB,CAA4C;IAE7D,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AACzG,CAAC"}

package/dist/storage/implementation/models.d.ts

@@ -1,4 +1,4 @@
-import { storage } from '@powersync/service-core';
+import { InternalOpId, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 /**
@@ -49,7 +49,7 @@ export interface BucketDataDocument {
     source_key?: ReplicaId;
     table?: string;
     row_id?: string;
-    checksum: number;
+    checksum: bigint;
     data: string | null;
     target_op?: bigint | null;
 }
@@ -68,6 +68,12 @@ export interface SourceTableDocument {
         type?: string;
     }[] | undefined;
     snapshot_done: boolean | undefined;
+    snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
+}
+export interface SourceTableDocumentSnapshotStatus {
+    total_estimated_count: number;
+    replicated_count: number;
+    last_key: bson.Binary | null;
 }
 /**
  * Record the state of each bucket.
@@ -99,6 +105,12 @@ export interface SyncRuleDocument {
      * Can only be false if state == PROCESSING.
      */
     snapshot_done: boolean;
+    /**
+     * If snapshot_done = false, this may be the lsn at which we started the snapshot.
+     *
+     * This can be used for resuming the snapshot after a restart.
+     */
+    snapshot_lsn: string | undefined;
     /**
      * The last consistent checkpoint.
      *
@@ -139,18 +151,34 @@ export interface SyncRuleDocument {
     last_fatal_error: string | null;
     content: string;
 }
+export interface CheckpointEventDocument {
+    _id: bson.ObjectId;
+}
 export type SyncRuleCheckpointState = Pick<SyncRuleDocument, 'last_checkpoint' | 'last_checkpoint_lsn' | '_id' | 'state'>;
 export interface CustomWriteCheckpointDocument {
     _id: bson.ObjectId;
     user_id: string;
     checkpoint: bigint;
     sync_rules_id: number;
+    /**
+     * Unlike managed write checkpoints, custom write checkpoints are flushed together with
+     * normal ops. This means we can assign an op_id for ordering / correlating with read checkpoints.
+     *
+     * This is not unique - multiple write checkpoints can have the same op_id.
+     */
+    op_id?: InternalOpId;
 }
 export interface WriteCheckpointDocument {
     _id: bson.ObjectId;
     user_id: string;
     lsns: Record<string, string>;
     client_id: bigint;
+    /**
+     * This is set to the checkpoint lsn when the checkpoint lsn >= this lsn.
+     * This is used to make it easier to determine what write checkpoints have been processed
+     * between two checkpoints.
+     */
+    processed_at_lsn: string | null;
 }
 export interface InstanceDocument {
     _id: string;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.9.5",
+  "version": "0.10.1",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -27,15 +27,15 @@
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-mongodb": "0.6.0",
-    "@powersync/lib-services-framework": "0.6.0",
-    "@powersync/service-core": "1.12.1",
+    "@powersync/lib-service-mongodb": "0.6.1",
+    "@powersync/lib-services-framework": "0.7.0",
+    "@powersync/service-core": "1.13.1",
     "@powersync/service-jsonbig": "0.17.10",
     "@powersync/service-sync-rules": "0.27.0",
-    "@powersync/service-types": "0.11.0"
+    "@powersync/service-types": "0.12.0"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.9.5"
+    "@powersync/service-core-tests": "0.10.1"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/db/migrations/1749720702136-checkpoint-events.ts

@@ -0,0 +1,50 @@
+import { migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.createCheckpointEventsCollection();
+
+    await db.write_checkpoints.createIndex(
+      {
+        processed_at_lsn: 1
+      },
+      { name: 'processed_at_lsn' }
+    );
+
+    await db.custom_write_checkpoints.createIndex(
+      {
+        op_id: 1
+      },
+      { name: 'op_id' }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    if (await db.write_checkpoints.indexExists('processed_at_lsn')) {
+      await db.write_checkpoints.dropIndex('processed_at_lsn');
+    }
+    if (await db.custom_write_checkpoints.indexExists('op_id')) {
+      await db.custom_write_checkpoints.dropIndex('op_id');
+    }
+    await db.db.dropCollection('checkpoint_events');
+  } finally {
+    await db.client.close();
+  }
+};

package/src/storage/MongoBucketStorage.ts

@@ -119,6 +119,7 @@ export class MongoBucketStorage
           }
         }
       );
+      await this.db.notifyCheckpoint();
     } else if (next == null && active?.id == sync_rules_group_id) {
       // Slot removed for "active" sync rules, while there is no "next" one.
       await this.updateSyncRules({
@@ -141,6 +142,7 @@ export class MongoBucketStorage
           }
         }
       );
+      await this.db.notifyCheckpoint();
     } else if (next != null && active?.id == sync_rules_group_id) {
       // Already have next sync rules, but need to stop replicating the active one.
 
@@ -155,6 +157,7 @@ export class MongoBucketStorage
           }
         }
       );
+      await this.db.notifyCheckpoint();
     }
   }
 
@@ -209,6 +212,7 @@ export class MongoBucketStorage
         no_checkpoint_before: null,
         keepalive_op: null,
         snapshot_done: false,
+        snapshot_lsn: undefined,
         state: storage.SyncRuleState.PROCESSING,
         slot_name: slot_name,
         last_checkpoint_ts: null,
@@ -216,6 +220,7 @@ export class MongoBucketStorage
         last_keepalive_ts: null
       };
       await this.db.sync_rules.insertOne(doc);
+      await this.db.notifyCheckpoint();
       rules = new MongoPersistedSyncRulesContent(this.db, doc);
       if (options.lock) {
         const lock = await rules.lock();