@powersync/service-module-mongodb-storage 0.0.0-dev-20250117095455 → 0.0.0-dev-20250214100224

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.0.0-dev-20250117095455",
+  "version": "0.0.0-dev-20250214100224",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -27,16 +27,16 @@
     "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "uuid": "^9.0.1",
-    "@powersync/lib-services-framework": "0.0.0-dev-20250117095455",
-    "@powersync/service-core": "0.0.0-dev-20250117095455",
+    "@powersync/lib-services-framework": "0.5.1",
+    "@powersync/service-core": "0.0.0-dev-20250214100224",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.23.1",
-    "@powersync/service-types": "0.0.0-dev-20250117095455",
-    "@powersync/lib-service-mongodb": "0.0.0-dev-20250117095455"
+    "@powersync/service-sync-rules": "0.23.4",
+    "@powersync/service-types": "0.8.0",
+    "@powersync/lib-service-mongodb": "0.4.1"
   },
   "devDependencies": {
     "@types/uuid": "^9.0.4",
-    "@powersync/service-core-tests": "0.0.0-dev-20250117095455"
+    "@powersync/service-core-tests": "0.0.0-dev-20250214100224"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -2,6 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { storage as core_storage, migrations } from '@powersync/service-core';
 import * as storage from '../../../storage/storage-index.js';
 import { MongoStorageConfig } from '../../../types/types.js';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   /**
@@ -65,7 +66,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
     const remaining = await db.sync_rules.find({ state: null as any }).toArray();
     if (remaining.length > 0) {
       const slots = remaining.map((doc) => doc.slot_name).join(', ');
-      throw new Error(`Invalid state for sync rules: ${slots}`);
+      throw new ServiceAssertionError(`Invalid state for sync rules: ${slots}`);
     }
   } finally {
     await db.client.close();

package/src/storage/MongoBucketStorage.ts

@@ -5,7 +5,7 @@ import * as timers from 'timers/promises';
 
 import { storage, sync, utils } from '@powersync/service-core';
 
-import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { DisposableObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 
 import * as lib_mongo from '@powersync/lib-service-mongodb';
@@ -84,22 +84,36 @@ export class MongoBucketStorage
     return storage;
   }
 
-  async configureSyncRules(sync_rules: string, options?: { lock?: boolean }) {
+  async getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier> {
+    const { setName: id } = await this.db.db.command({
+      hello: 1
+    });
+    if (id == null) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1342,
+        'Standalone MongoDB instances are not supported - use a replicaset.'
+      );
+    }
+
+    return {
+      id,
+      type: lib_mongo.MONGO_CONNECTION_TYPE
+    };
+  }
+
+  async configureSyncRules(options: storage.UpdateSyncRulesOptions) {
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();
 
-    if (next?.sync_rules_content == sync_rules) {
+    if (next?.sync_rules_content == options.content) {
       logger.info('Sync rules from configuration unchanged');
       return { updated: false };
-    } else if (next == null && active?.sync_rules_content == sync_rules) {
+    } else if (next == null && active?.sync_rules_content == options.content) {
       logger.info('Sync rules from configuration unchanged');
       return { updated: false };
     } else {
       logger.info('Sync rules updated from configuration');
-      const persisted_sync_rules = await this.updateSyncRules({
-        content: sync_rules,
-        lock: options?.lock
-      });
+      const persisted_sync_rules = await this.updateSyncRules(options);
       return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
     }
   }
@@ -113,7 +127,8 @@ export class MongoBucketStorage
     if (next != null && next.slot_name == slot_name) {
       // We need to redo the "next" sync rules
       await this.updateSyncRules({
-        content: next.sync_rules_content
+        content: next.sync_rules_content,
+        validate: false
       });
       // Pro-actively stop replicating
       await this.db.sync_rules.updateOne(
@@ -130,7 +145,8 @@ export class MongoBucketStorage
     } else if (next == null && active?.slot_name == slot_name) {
       // Slot removed for "active" sync rules, while there is no "next" one.
       await this.updateSyncRules({
-        content: active.sync_rules_content
+        content: active.sync_rules_content,
+        validate: false
       });
 
       // Pro-actively stop replicating
@@ -149,13 +165,18 @@ export class MongoBucketStorage
   }
 
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
-    // Parse and validate before applying any changes
-    const parsed = SqlSyncRules.fromYaml(options.content, {
-      // No schema-based validation at this point
-      schema: undefined,
-      defaultSchema: 'not_applicable', // Not needed for validation
-      throwOnError: true
-    });
+    if (options.validate) {
+      // Parse and validate before applying any changes
+      SqlSyncRules.fromYaml(options.content, {
+        // No schema-based validation at this point
+        schema: undefined,
+        defaultSchema: 'not_applicable', // Not needed for validation
+        throwOnError: true
+      });
+    } else {
+      // We do not validate sync rules at this point.
+      // That is done when using the sync rules, so that the diagnostics API can report the errors.
+    }
 
     let rules: MongoPersistedSyncRulesContent | undefined = undefined;
 
@@ -433,7 +454,7 @@ export class MongoBucketStorage
       clusterTime = time;
     });
     if (clusterTime == null) {
-      throw new Error('Could not get clusterTime');
+      throw new ServiceError(ErrorCode.PSYNC_S2401, 'Could not get clusterTime');
     }
 
     if (signal.aborted) {

package/src/storage/implementation/MongoBucketBatch.ts

@@ -2,7 +2,15 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 
-import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework';
+import {
+  container,
+  DisposableObserver,
+  ErrorCode,
+  errors,
+  logger,
+  ReplicationAssertionError,
+  ServiceError
+} from '@powersync/lib-services-framework';
 import { SaveOperationTag, storage, utils } from '@powersync/service-core';
 import * as timers from 'node:timers/promises';
 import { PowerSyncMongo } from './db.js';
@@ -140,7 +148,7 @@ export class MongoBucketBatch
     this.batch = resumeBatch;
 
     if (last_op == null) {
-      throw new Error('Unexpected last_op == null');
+      throw new ReplicationAssertionError('Unexpected last_op == null');
     }
 
     this.persisted_op = last_op;
@@ -294,7 +302,7 @@ export class MongoBucketBatch
           return null;
         }
       } else {
-        throw new Error(`${record.tag} not supported with skipExistingRows: true`);
+        throw new ReplicationAssertionError(`${record.tag} not supported with skipExistingRows: true`);
       }
     }
 
@@ -348,7 +356,7 @@ export class MongoBucketBatch
         afterData = new bson.Binary(bson.serialize(after!));
         // We additionally make sure it's <= 15MB - we need some margin for metadata.
         if (afterData.length() > MAX_ROW_SIZE) {
-          throw new Error(`Row too large: ${afterData.length()}`);
+          throw new ServiceError(ErrorCode.PSYNC_S1002, `Row too large: ${afterData.length()}`);
         }
       } catch (e) {
         // Replace with empty values, equivalent to TOAST values
@@ -548,7 +556,7 @@ export class MongoBucketBatch
         logger.info(`${this.slot_name} ${description} - try ${flushTry}`);
       }
       if (flushTry > 20 && Date.now() > lastTry) {
-        throw new Error('Max transaction tries exceeded');
+        throw new ServiceError(ErrorCode.PSYNC_S1402, 'Max transaction tries exceeded');
       }
 
       const next_op_id_doc = await this.db.op_id_sequence.findOneAndUpdate(
@@ -607,7 +615,9 @@ export class MongoBucketBatch
 
   private lastWaitingLogThottled = 0;
 
-  async commit(lsn: string): Promise<boolean> {
+  async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<boolean> {
+    const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
+
     await this.flush();
 
     if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
@@ -645,6 +655,10 @@ export class MongoBucketBatch
       return false;
     }
 
+    if (!createEmptyCheckpoints && this.persisted_op == null) {
+      return false;
+    }
+
     const now = new Date();
     const update: Partial<SyncRuleDocument> = {
       last_checkpoint_lsn: lsn,

package/src/storage/implementation/MongoCompactor.ts

@@ -1,5 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { logger } from '@powersync/lib-services-framework';
+import { logger, ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 
 import { PowerSyncMongo } from './db.js';
@@ -335,7 +335,9 @@ export class MongoCompactor {
                   }
                 }
               } else {
-                throw new Error(`Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}`);
+                throw new ReplicationAssertionError(
+                  `Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}`
+                );
               }
             }
             if (!gotAnOp) {

package/src/storage/implementation/MongoIdSequence.ts

@@ -1,3 +1,5 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+
 /**
  * Manages op_id or similar sequence in memory.
  *
@@ -9,7 +11,7 @@ export class MongoIdSequence {
 
   constructor(last: bigint) {
     if (typeof last != 'bigint') {
-      throw new Error(`BigInt required, got ${last} ${typeof last}`);
+      throw new ReplicationAssertionError(`BigInt required, got ${last} ${typeof last}`);
     }
     this._last = last;
   }

package/src/storage/implementation/MongoStorageProvider.ts

@@ -1,5 +1,5 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { logger } from '@powersync/lib-services-framework';
+import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { MongoStorageConfig } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
@@ -16,7 +16,9 @@ export class MongoStorageProvider implements storage.BucketStorageProvider {
     const { storage } = resolvedConfig;
     if (storage.type != this.type) {
       // This should not be reached since the generation should be managed externally.
-      throw new Error(`Cannot create MongoDB bucket storage with provided config ${storage.type} !== ${this.type}`);
+      throw new ServiceAssertionError(
+        `Cannot create MongoDB bucket storage with provided config ${storage.type} !== ${this.type}`
+      );
     }
 
     const decodedConfig = MongoStorageConfig.decode(storage as any);

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -1,6 +1,6 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
-import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { DisposableObserver, logger, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
@@ -344,7 +344,7 @@ export class MongoSyncBucketStorage
 
         start ??= dataBuckets.get(bucket);
         if (start == null) {
-          throw new Error(`data for unexpected bucket: ${bucket}`);
+          throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
         }
         currentBatch = {
           bucket,
@@ -479,7 +479,7 @@ export class MongoSyncBucketStorage
       }
     );
     if (doc == null) {
-      throw new Error('Cannot find sync rules status');
+      throw new ServiceAssertionError('Cannot find sync rules status');
     }
 
     return {

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -1,6 +1,6 @@
 import crypto from 'crypto';
 
-import { logger } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
 
@@ -33,7 +33,10 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
     );
 
     if (doc == null) {
-      throw new Error(`Sync rules: ${sync_rules.id} have been locked by another process for replication.`);
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1003,
+        `Sync rules: ${sync_rules.id} have been locked by another process for replication.`
+      );
     }
     return new MongoSyncRulesLock(db, sync_rules.id, lockId);
   }

package/test/src/storage_compacting.test.ts

@@ -3,4 +3,9 @@ import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
-describe('Mongo Sync Bucket Storage Compact', () => register.registerCompactTests<MongoCompactOptions>(INITIALIZED_MONGO_STORAGE_FACTORY, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
+describe('Mongo Sync Bucket Storage Compact', () =>
+  register.registerCompactTests<MongoCompactOptions>(INITIALIZED_MONGO_STORAGE_FACTORY, {
+    clearBatchLimit: 2,
+    moveBatchLimit: 1,
+    moveBatchQueryLimit: 1
+  }));