@powersync/service-module-mongodb-storage 0.13.2 → 0.15.0

package/src/storage/MongoBucketStorage.ts

@@ -1,23 +1,23 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-
 import { GetIntanceOptions, storage } from '@powersync/service-core';
 
-import { BaseObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 
 import { PowerSyncMongo } from './implementation/db.js';
-import { SyncRuleDocument } from './implementation/models.js';
+import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js';
 import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
-import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './implementation/MongoSyncBucketStorage.js';
+import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
 import { generateSlotName } from '../utils/util.js';
+import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
+
+export interface MongoBucketStorageOptions {
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+}
 
-export class MongoBucketStorage
-  extends BaseObserver<storage.BucketStorageFactoryListener>
-  implements storage.BucketStorageFactory
-{
+export class MongoBucketStorage extends storage.BucketStorageFactory {
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
   // TODO: This is still Postgres specific and needs to be reworked
@@ -32,7 +32,7 @@ export class MongoBucketStorage
     options: {
       slot_name_prefix: string;
     },
-    private internalOptions?: MongoSyncBucketStorageOptions
+    private internalOptions?: MongoBucketStorageOptions
   ) {
     super();
     this.client = db.client;
@@ -50,10 +50,22 @@ export class MongoBucketStorage
     if ((typeof id as any) == 'bigint') {
       id = Number(id);
     }
-    const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name, undefined, this.internalOptions);
+    const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig();
+    const storage = new MongoSyncBucketStorage(
+      this,
+      id,
+      syncRules as MongoPersistedSyncRulesContent,
+      slot_name,
+      undefined,
+      {
+        ...this.internalOptions,
+        storageConfig
+      }
+    );
     if (!options?.skipLifecycleHooks) {
       this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
     }
+
     storage.registerListener({
       batchStarted: (batch) => {
         batch.registerListener({
@@ -81,33 +93,13 @@ export class MongoBucketStorage
     };
   }
 
-  async configureSyncRules(options: storage.UpdateSyncRulesOptions) {
-    const next = await this.getNextSyncRulesContent();
-    const active = await this.getActiveSyncRulesContent();
-
-    if (next?.sync_rules_content == options.content) {
-      logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else if (next == null && active?.sync_rules_content == options.content) {
-      logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else {
-      logger.info('Sync rules updated from configuration');
-      const persisted_sync_rules = await this.updateSyncRules(options);
-      return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
-    }
-  }
-
   async restartReplication(sync_rules_group_id: number) {
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();
 
     if (next != null && next.id == sync_rules_group_id) {
       // We need to redo the "next" sync rules
-      await this.updateSyncRules({
-        content: next.sync_rules_content,
-        validate: false
-      });
+      await this.updateSyncRules(next.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sync_rules.updateOne(
         {
@@ -123,10 +115,7 @@ export class MongoBucketStorage
       await this.db.notifyCheckpoint();
     } else if (next == null && active?.id == sync_rules_group_id) {
       // Slot removed for "active" sync rules, while there is no "next" one.
-      await this.updateSyncRules({
-        content: active.sync_rules_content,
-        validate: false
-      });
+      await this.updateSyncRules(active.asUpdateOptions());
 
       // In this case we keep the old one as active for clients, so that that existing clients
       // can still get the latest data while we replicate the new ones.
@@ -163,18 +152,9 @@ export class MongoBucketStorage
   }
 
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
-    if (options.validate) {
-      // Parse and validate before applying any changes
-      SqlSyncRules.fromYaml(options.content, {
-        // No schema-based validation at this point
-        schema: undefined,
-        defaultSchema: 'not_applicable', // Not needed for validation
-        throwOnError: true
-      });
-    } else {
-      // We do not validate sync rules at this point.
-      // That is done when using the sync rules, so that the diagnostics API can report the errors.
-    }
+    const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+    const storageConfig = getMongoStorageConfig(storageVersion);
+    await this.db.initializeStorageVersion(storageConfig);
 
     let rules: MongoPersistedSyncRulesContent | undefined = undefined;
 
@@ -207,7 +187,9 @@ export class MongoBucketStorage
 
       const doc: SyncRuleDocument = {
         _id: id,
-        content: options.content,
+        storage_version: storageVersion,
+        content: options.config.yaml,
+        serialized_plan: options.config.plan,
         last_checkpoint: null,
         last_checkpoint_lsn: null,
         no_checkpoint_before: null,
@@ -246,11 +228,6 @@ export class MongoBucketStorage
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getActiveSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getActiveSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
     const doc = await this.db.sync_rules.findOne(
       {
@@ -265,11 +242,6 @@ export class MongoBucketStorage
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getNextSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getNextSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
     const docs = await this.db.sync_rules
       .find({
@@ -353,7 +325,7 @@ export class MongoBucketStorage
       .toArray()
       .catch(ignoreNotExisting);
 
-    const replication_aggregate = await this.db.current_data
+    const v1_replication_aggregate = await this.db.current_data
       .aggregate([
         {
           $collStats: {
@@ -364,10 +336,21 @@ export class MongoBucketStorage
       .toArray()
       .catch(ignoreNotExisting);
 
+    const v3_replication_aggregate = await this.db.v3_current_data
+      .aggregate([
+        {
+          $collStats: {
+            storageStats: {}
+          }
+        }
+      ])
+      .toArray()
+      .catch(ignoreNotExisting);
     return {
       operations_size_bytes: Number(operations_aggregate[0].storageStats.size),
       parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size),
-      replication_size_bytes: Number(replication_aggregate[0].storageStats.size)
+      replication_size_bytes:
+        Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size)
     };
   }