@powersync/service-module-mongodb-storage 0.2.0 → 0.3.1

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -27,16 +27,16 @@
     "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "uuid": "^9.0.1",
-    "@powersync/lib-services-framework": "0.3.0",
-    "@powersync/service-core": "0.14.0",
+    "@powersync/lib-services-framework": "0.5.0",
+    "@powersync/service-core": "0.16.0",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.23.1",
-    "@powersync/service-types": "0.7.0",
-    "@powersync/lib-service-mongodb": "0.3.0"
+    "@powersync/service-sync-rules": "0.23.2",
+    "@powersync/service-types": "0.7.1",
+    "@powersync/lib-service-mongodb": "0.4.0"
   },
   "devDependencies": {
     "@types/uuid": "^9.0.4",
-    "@powersync/service-core-tests": "0.2.0"
+    "@powersync/service-core-tests": "0.3.1"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -2,6 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { storage as core_storage, migrations } from '@powersync/service-core';
 import * as storage from '../../../storage/storage-index.js';
 import { MongoStorageConfig } from '../../../types/types.js';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   /**
@@ -65,7 +66,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
     const remaining = await db.sync_rules.find({ state: null as any }).toArray();
     if (remaining.length > 0) {
       const slots = remaining.map((doc) => doc.slot_name).join(', ');
-      throw new Error(`Invalid state for sync rules: ${slots}`);
+      throw new ServiceAssertionError(`Invalid state for sync rules: ${slots}`);
     }
   } finally {
     await db.client.close();

package/src/storage/MongoBucketStorage.ts

@@ -5,7 +5,7 @@ import * as timers from 'timers/promises';
 
 import { storage, sync, utils } from '@powersync/service-core';
 
-import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { DisposableObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 
 import * as lib_mongo from '@powersync/lib-service-mongodb';
@@ -62,6 +62,10 @@ export class MongoBucketStorage
     this.slot_name_prefix = options.slot_name_prefix;
   }
 
+  async [Symbol.asyncDispose]() {
+    super[Symbol.dispose]();
+  }
+
   getInstance(options: storage.PersistedSyncRulesContent): MongoSyncBucketStorage {
     let { id, slot_name } = options;
     if ((typeof id as any) == 'bigint') {
@@ -106,7 +110,6 @@ export class MongoBucketStorage
 
     // In both the below cases, we create a new sync rules instance.
     // The current one will continue erroring until the next one has finished processing.
-    // TODO: Update
     if (next != null && next.slot_name == slot_name) {
       // We need to redo the "next" sync rules
       await this.updateSyncRules({
@@ -430,7 +433,7 @@ export class MongoBucketStorage
       clusterTime = time;
     });
     if (clusterTime == null) {
-      throw new Error('Could not get clusterTime');
+      throw new ServiceError(ErrorCode.PSYNC_S2401, 'Could not get clusterTime');
     }
 
     if (signal.aborted) {

package/src/storage/implementation/MongoBucketBatch.ts

@@ -2,7 +2,16 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 
-import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework';
+import {
+  container,
+  DisposableObserver,
+  ErrorCode,
+  errors,
+  logger,
+  ReplicationAssertionError,
+  ServiceAssertionError,
+  ServiceError
+} from '@powersync/lib-services-framework';
 import { SaveOperationTag, storage, utils } from '@powersync/service-core';
 import * as timers from 'node:timers/promises';
 import { PowerSyncMongo } from './db.js';
@@ -140,7 +149,7 @@ export class MongoBucketBatch
     this.batch = resumeBatch;
 
     if (last_op == null) {
-      throw new Error('Unexpected last_op == null');
+      throw new ReplicationAssertionError('Unexpected last_op == null');
     }
 
     this.persisted_op = last_op;
@@ -294,7 +303,7 @@ export class MongoBucketBatch
           return null;
         }
       } else {
-        throw new Error(`${record.tag} not supported with skipExistingRows: true`);
+        throw new ReplicationAssertionError(`${record.tag} not supported with skipExistingRows: true`);
       }
     }
 
@@ -348,7 +357,7 @@ export class MongoBucketBatch
         afterData = new bson.Binary(bson.serialize(after!));
         // We additionally make sure it's <= 15MB - we need some margin for metadata.
         if (afterData.length() > MAX_ROW_SIZE) {
-          throw new Error(`Row too large: ${afterData.length()}`);
+          throw new ServiceError(ErrorCode.PSYNC_S1002, `Row too large: ${afterData.length()}`);
         }
       } catch (e) {
         // Replace with empty values, equivalent to TOAST values
@@ -548,7 +557,7 @@ export class MongoBucketBatch
         logger.info(`${this.slot_name} ${description} - try ${flushTry}`);
       }
       if (flushTry > 20 && Date.now() > lastTry) {
-        throw new Error('Max transaction tries exceeded');
+        throw new ServiceError(ErrorCode.PSYNC_S1402, 'Max transaction tries exceeded');
       }
 
       const next_op_id_doc = await this.db.op_id_sequence.findOneAndUpdate(

package/src/storage/implementation/MongoCompactor.ts

@@ -1,5 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { logger } from '@powersync/lib-services-framework';
+import { logger, ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 
 import { PowerSyncMongo } from './db.js';
@@ -335,7 +335,9 @@ export class MongoCompactor {
                   }
                 }
               } else {
-                throw new Error(`Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}`);
+                throw new ReplicationAssertionError(
+                  `Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}`
+                );
               }
             }
             if (!gotAnOp) {

package/src/storage/implementation/MongoIdSequence.ts

@@ -1,3 +1,5 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+
 /**
  * Manages op_id or similar sequence in memory.
  *
@@ -9,7 +11,7 @@ export class MongoIdSequence {
 
   constructor(last: bigint) {
     if (typeof last != 'bigint') {
-      throw new Error(`BigInt required, got ${last} ${typeof last}`);
+      throw new ReplicationAssertionError(`BigInt required, got ${last} ${typeof last}`);
     }
     this._last = last;
   }

package/src/storage/implementation/MongoStorageProvider.ts

@@ -1,5 +1,5 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { logger } from '@powersync/lib-services-framework';
+import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { MongoStorageConfig } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
@@ -16,20 +16,25 @@ export class MongoStorageProvider implements storage.BucketStorageProvider {
     const { storage } = resolvedConfig;
     if (storage.type != this.type) {
       // This should not be reached since the generation should be managed externally.
-      throw new Error(`Cannot create MongoDB bucket storage with provided config ${storage.type} !== ${this.type}`);
+      throw new ServiceAssertionError(
+        `Cannot create MongoDB bucket storage with provided config ${storage.type} !== ${this.type}`
+      );
     }
 
     const decodedConfig = MongoStorageConfig.decode(storage as any);
     const client = lib_mongo.db.createMongoClient(decodedConfig);
 
     const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
-
+    const factory = new MongoBucketStorage(database, {
+      // TODO currently need the entire resolved config due to this
+      slot_name_prefix: resolvedConfig.slot_name_prefix
+    });
     return {
-      storage: new MongoBucketStorage(database, {
-        // TODO currently need the entire resolved config due to this
-        slot_name_prefix: resolvedConfig.slot_name_prefix
-      }),
-      shutDown: () => client.close(),
+      storage: factory,
+      shutDown: async () => {
+        await factory[Symbol.asyncDispose]();
+        await client.close();
+      },
       tearDown: () => {
         logger.info(`Tearing down storage: ${database.db.namespace}...`);
         return database.db.dropDatabase();

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -1,6 +1,6 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
-import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { DisposableObserver, logger, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
@@ -344,7 +344,7 @@ export class MongoSyncBucketStorage
 
         start ??= dataBuckets.get(bucket);
         if (start == null) {
-          throw new Error(`data for unexpected bucket: ${bucket}`);
+          throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
         }
         currentBatch = {
           bucket,
@@ -479,7 +479,7 @@ export class MongoSyncBucketStorage
       }
     );
     if (doc == null) {
-      throw new Error('Cannot find sync rules status');
+      throw new ServiceAssertionError('Cannot find sync rules status');
     }
 
     return {

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -1,6 +1,6 @@
 import crypto from 'crypto';
 
-import { logger } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
 
@@ -33,7 +33,10 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
     );
 
     if (doc == null) {
-      throw new Error(`Sync rules: ${sync_rules.id} have been locked by another process for replication.`);
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1003,
+        `Sync rules: ${sync_rules.id} have been locked by another process for replication.`
+      );
     }
     return new MongoSyncRulesLock(db, sync_rules.id, lockId);
   }

package/test/src/__snapshots__/storage.test.ts.snap

@@ -0,0 +1,9 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Mongo Sync Bucket Storage > empty storage metrics 1`] = `
+{
+  "operations_size_bytes": 0,
+  "parameters_size_bytes": 0,
+  "replication_size_bytes": 0,
+}
+`;

package/test/src/migrations.test.ts

@@ -0,0 +1,10 @@
+import { register } from '@powersync/service-core-tests';
+import { describe } from 'vitest';
+import { MongoMigrationAgent } from '../../src/migrations/MongoMigrationAgent.js';
+import { env } from './env.js';
+
+const MIGRATION_AGENT_FACTORY = () => {
+  return new MongoMigrationAgent({ type: 'mongodb', uri: env.MONGO_TEST_URL });
+};
+
+describe('Mongo Migrations Store', () => register.registerMigrationTests(MIGRATION_AGENT_FACTORY));

package/test/src/setup.ts

@@ -1,9 +1,13 @@
 import { container } from '@powersync/lib-services-framework';
 import { test_utils } from '@powersync/service-core-tests';
-import { beforeAll } from 'vitest';
+import { beforeAll, beforeEach } from 'vitest';
 
 beforeAll(async () => {
   // Executes for every test file
   container.registerDefaults();
   await test_utils.initMetrics();
 });
+
+beforeEach(async () => {
+  await test_utils.resetMetrics();
+});

package/test/src/storage_compacting.test.ts

@@ -3,4 +3,9 @@ import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
-describe('Mongo Sync Bucket Storage Compact', () => register.registerCompactTests<MongoCompactOptions>(INITIALIZED_MONGO_STORAGE_FACTORY, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
+describe('Mongo Sync Bucket Storage Compact', () =>
+  register.registerCompactTests<MongoCompactOptions>(INITIALIZED_MONGO_STORAGE_FACTORY, {
+    clearBatchLimit: 2,
+    moveBatchLimit: 1,
+    moveBatchQueryLimit: 1
+  }));