@powersync/service-module-mongodb-storage 0.13.1 → 0.14.0

package/src/storage/MongoBucketStorage.ts

@@ -1,8 +1,6 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-
 import { GetIntanceOptions, storage } from '@powersync/service-core';
 
-import { BaseObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 
 import * as lib_mongo from '@powersync/lib-service-mongodb';
@@ -11,13 +9,15 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { PowerSyncMongo } from './implementation/db.js';
 import { SyncRuleDocument } from './implementation/models.js';
 import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
-import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './implementation/MongoSyncBucketStorage.js';
+import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
 import { generateSlotName } from '../utils/util.js';
+import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
+
+export interface MongoBucketStorageOptions {
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+}
 
-export class MongoBucketStorage
-  extends BaseObserver<storage.BucketStorageFactoryListener>
-  implements storage.BucketStorageFactory
-{
+export class MongoBucketStorage extends storage.BucketStorageFactory {
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
   // TODO: This is still Postgres specific and needs to be reworked
@@ -32,7 +32,7 @@ export class MongoBucketStorage
     options: {
       slot_name_prefix: string;
     },
-    private internalOptions?: MongoSyncBucketStorageOptions
+    private internalOptions?: MongoBucketStorageOptions
   ) {
     super();
     this.client = db.client;
@@ -50,10 +50,15 @@ export class MongoBucketStorage
     if ((typeof id as any) == 'bigint') {
       id = Number(id);
     }
-    const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name, undefined, this.internalOptions);
+    const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig();
+    const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name, undefined, {
+      ...this.internalOptions,
+      storageConfig
+    });
     if (!options?.skipLifecycleHooks) {
       this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
     }
+
     storage.registerListener({
       batchStarted: (batch) => {
         batch.registerListener({
@@ -81,33 +86,13 @@ export class MongoBucketStorage
     };
   }
 
-  async configureSyncRules(options: storage.UpdateSyncRulesOptions) {
-    const next = await this.getNextSyncRulesContent();
-    const active = await this.getActiveSyncRulesContent();
-
-    if (next?.sync_rules_content == options.content) {
-      logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else if (next == null && active?.sync_rules_content == options.content) {
-      logger.info('Sync rules from configuration unchanged');
-      return { updated: false };
-    } else {
-      logger.info('Sync rules updated from configuration');
-      const persisted_sync_rules = await this.updateSyncRules(options);
-      return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
-    }
-  }
-
   async restartReplication(sync_rules_group_id: number) {
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();
 
     if (next != null && next.id == sync_rules_group_id) {
       // We need to redo the "next" sync rules
-      await this.updateSyncRules({
-        content: next.sync_rules_content,
-        validate: false
-      });
+      await this.updateSyncRules(next.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sync_rules.updateOne(
         {
@@ -123,10 +108,7 @@ export class MongoBucketStorage
       await this.db.notifyCheckpoint();
     } else if (next == null && active?.id == sync_rules_group_id) {
       // Slot removed for "active" sync rules, while there is no "next" one.
-      await this.updateSyncRules({
-        content: active.sync_rules_content,
-        validate: false
-      });
+      await this.updateSyncRules(active.asUpdateOptions());
 
       // In this case we keep the old one as active for clients, so that that existing clients
       // can still get the latest data while we replicate the new ones.
@@ -163,19 +145,6 @@ export class MongoBucketStorage
   }
 
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
-    if (options.validate) {
-      // Parse and validate before applying any changes
-      SqlSyncRules.fromYaml(options.content, {
-        // No schema-based validation at this point
-        schema: undefined,
-        defaultSchema: 'not_applicable', // Not needed for validation
-        throwOnError: true
-      });
-    } else {
-      // We do not validate sync rules at this point.
-      // That is done when using the sync rules, so that the diagnostics API can report the errors.
-    }
-
     let rules: MongoPersistedSyncRulesContent | undefined = undefined;
 
     await this.session.withTransaction(async () => {
@@ -205,9 +174,12 @@ export class MongoBucketStorage
       const id = Number(id_doc!.op_id);
       const slot_name = generateSlotName(this.slot_name_prefix, id);
 
+      const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
       const doc: SyncRuleDocument = {
         _id: id,
-        content: options.content,
+        storage_version: storageVersion,
+        content: options.config.yaml,
+        serialized_plan: options.config.plan,
         last_checkpoint: null,
         last_checkpoint_lsn: null,
         no_checkpoint_before: null,
@@ -246,11 +218,6 @@ export class MongoBucketStorage
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getActiveSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getActiveSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
     const doc = await this.db.sync_rules.findOne(
       {
@@ -265,11 +232,6 @@ export class MongoBucketStorage
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getNextSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null> {
-    const content = await this.getNextSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
-
   async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
     const docs = await this.db.sync_rules
       .find({

package/src/storage/implementation/MongoChecksums.ts

@@ -13,6 +13,7 @@ import {
   PartialOrFullChecksum
 } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
+import { StorageConfig } from './models.js';
 
 /**
  * Checksum calculation options, primarily for tests.
@@ -27,6 +28,8 @@ export interface MongoChecksumOptions {
    * Limit on the number of documents to calculate a checksum on at a time.
    */
   operationBatchLimit?: number;
+
+  storageConfig: StorageConfig;
 }
 
 const DEFAULT_BUCKET_BATCH_LIMIT = 200;
@@ -43,12 +46,15 @@ const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
  */
 export class MongoChecksums {
   private _cache: ChecksumCache | undefined;
+  private readonly storageConfig: StorageConfig;
 
   constructor(
     private db: PowerSyncMongo,
     private group_id: number,
-    private options?: MongoChecksumOptions
-  ) {}
+    private options: MongoChecksumOptions
+  ) {
+    this.storageConfig = options.storageConfig;
+  }
 
   /**
    * Lazy-instantiated cache.
@@ -222,6 +228,11 @@ export class MongoChecksums {
         });
       }
 
+      // Historically, checksum may be stored as 'int' or 'double'.
+      // More recently, this should be a 'long'.
+      // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
+      const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' };
+
       // Aggregate over a max of `batchLimit` operations at a time.
       // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
       // Then we'll do three batches:
@@ -245,10 +256,7 @@ export class MongoChecksums {
             {
               $group: {
                 _id: '$_id.b',
-                // Historically, checksum may be stored as 'int' or 'double'.
-                // More recently, this should be a 'long'.
-                // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-                checksum_total: { $sum: { $toLong: '$checksum' } },
+                checksum_total: { $sum: checksumLong },
                 count: { $sum: 1 },
                 has_clear_op: {
                   $max: {

package/src/storage/implementation/MongoCompactor.ts

@@ -63,6 +63,7 @@ const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 const DEFAULT_MIN_BUCKET_CHANGES = 10;
 const DEFAULT_MIN_CHANGE_RATIO = 0.1;
+const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
 
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
@@ -485,19 +486,14 @@ export class MongoCompactor {
    */
   async populateChecksums(options: { minBucketChanges: number }): Promise<PopulateChecksumCacheResults> {
     let count = 0;
-    for await (let buckets of this.dirtyBucketBatches({
-      minBucketChanges: options.minBucketChanges,
-      minChangeRatio: 0
-    })) {
-      if (this.signal?.aborted) {
+    while (!this.signal?.aborted) {
+      const buckets = await this.dirtyBucketBatchForChecksums(options);
+      if (buckets.length == 0 || this.signal?.aborted) {
+        // All done
         break;
       }
-      if (buckets.length == 0) {
-        continue;
-      }
 
       const start = Date.now();
-      logger.info(`Calculating checksums for batch of ${buckets.length} buckets, starting at ${buckets[0].bucket}`);
 
       // Filter batch by estimated bucket size, to reduce possibility of timeouts
       let checkBuckets: typeof buckets = [];
@@ -509,9 +505,12 @@ export class MongoCompactor {
           break;
         }
       }
+      logger.info(
+        `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`
+      );
       await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket));
       logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
-      count += buckets.length;
+      count += checkBuckets.length;
     }
     return { buckets: count };
   }
@@ -540,31 +539,60 @@ export class MongoCompactor {
     let lastId = { g: this.group_id, b: new mongo.MinKey() as any };
     const maxId = { g: this.group_id, b: new mongo.MaxKey() as any };
     while (true) {
-      const batch = await this.db.bucket_state
-        .find(
-          {
-            _id: { $gt: lastId, $lt: maxId },
-            'estimate_since_compact.count': { $gte: options.minBucketChanges }
-          },
-          {
-            projection: {
-              _id: 1,
-              estimate_since_compact: 1,
-              compacted_state: 1
+      // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, we use an aggregation pipeline
+      // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria, rather than limiting
+      // on the output number.
+      const [result] = await this.db.bucket_state
+        .aggregate<{
+          buckets: Pick<BucketStateDocument, '_id' | 'estimate_since_compact' | 'compacted_state'>[];
+          cursor: Pick<BucketStateDocument, '_id'>[];
+        }>(
+          [
+            {
+              $match: {
+                _id: { $gt: lastId, $lt: maxId }
+              }
             },
-            sort: {
-              _id: 1
+            {
+              $sort: { _id: 1 }
             },
-            limit: 2000,
-            maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
-          }
+            {
+              // Scan a fixed number of docs each query so sparse matches don't block progress.
+              $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE
+            },
+            {
+              $facet: {
+                // This is the results for the batch
+                buckets: [
+                  {
+                    $match: {
+                      'estimate_since_compact.count': { $gte: options.minBucketChanges }
+                    }
+                  },
+                  {
+                    $project: {
+                      _id: 1,
+                      estimate_since_compact: 1,
+                      compacted_state: 1
+                    }
+                  }
+                ],
+                // This is used for the next query.
+                cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
+              }
+            }
+          ],
+          { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS }
         )
         .toArray();
-      if (batch.length == 0) {
+
+      const cursor = result?.cursor?.[0];
+      if (cursor == null) {
         break;
       }
-      lastId = batch[batch.length - 1]._id;
-      const mapped = batch.map((b) => {
+      lastId = cursor._id;
+
+      const mapped = (result?.buckets ?? []).map((b) => {
         const updatedCount = b.estimate_since_compact?.count ?? 0;
         const totalCount = (b.compacted_state?.count ?? 0) + updatedCount;
         const updatedBytes = b.estimate_since_compact?.bytes ?? 0;
@@ -584,6 +612,51 @@ export class MongoCompactor {
     }
   }
 
+  /**
+   * Returns a batch of dirty buckets - buckets with most changes first.
+   *
+   * This cannot be used to iterate on its own - the client is expected to process these buckets and
+   * set estimate_since_compact.count: 0 when done, before fetching the next batch.
+   *
+   * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart,
+   * since it is used as the last step for initial replication.
+   *
+   * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing.
+   */
+  private async dirtyBucketBatchForChecksums(options: {
+    minBucketChanges: number;
+  }): Promise<{ bucket: string; estimatedCount: number }[]> {
+    if (options.minBucketChanges <= 0) {
+      throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    }
+    // We make use of an index on {_id.g: 1, 'estimate_since_compact.count': -1}
+    const dirtyBuckets = await this.db.bucket_state
+      .find(
+        {
+          '_id.g': this.group_id,
+          'estimate_since_compact.count': { $gte: options.minBucketChanges }
+        },
+        {
+          projection: {
+            _id: 1,
+            estimate_since_compact: 1,
+            compacted_state: 1
+          },
+          sort: {
+            'estimate_since_compact.count': -1
+          },
+          limit: 200,
+          maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
+        }
+      )
+      .toArray();
+
+    return dirtyBuckets.map((bucket) => ({
+      bucket: bucket._id.b,
+      estimatedCount: bucket.estimate_since_compact!.count + (bucket.compacted_state?.count ?? 0)
+    }));
+  }
+
   private async updateChecksumsBatch(buckets: string[]) {
     const checksums = await this.storage.checksums.computePartialChecksumsDirect(
       buckets.map((bucket) => {

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -1,48 +1,42 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
 import { PowerSyncMongo } from './db.js';
-import { SyncRuleDocument } from './models.js';
-
-export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRulesContent {
-  public readonly slot_name: string;
-
-  public readonly id: number;
-  public readonly sync_rules_content: string;
-  public readonly last_checkpoint_lsn: string | null;
-  public readonly last_fatal_error: string | null;
-  public readonly last_fatal_error_ts: Date | null;
-  public readonly last_keepalive_ts: Date | null;
-  public readonly last_checkpoint_ts: Date | null;
-  public readonly active: boolean;
+import { getMongoStorageConfig, SyncRuleDocument } from './models.js';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 
+export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
   public current_lock: MongoSyncRulesLock | null = null;
 
   constructor(
     private db: PowerSyncMongo,
     doc: mongo.WithId<SyncRuleDocument>
   ) {
-    this.id = doc._id;
-    this.sync_rules_content = doc.content;
-    this.last_checkpoint_lsn = doc.last_checkpoint_lsn;
-    // Handle legacy values
-    this.slot_name = doc.slot_name ?? `powersync_${this.id}`;
-    this.last_fatal_error = doc.last_fatal_error;
-    this.last_fatal_error_ts = doc.last_fatal_error_ts;
-    this.last_checkpoint_ts = doc.last_checkpoint_ts;
-    this.last_keepalive_ts = doc.last_keepalive_ts;
-    this.active = doc.state == 'ACTIVE';
+    super({
+      id: doc._id,
+      sync_rules_content: doc.content,
+      compiled_plan: doc.serialized_plan ?? null,
+      last_checkpoint_lsn: doc.last_checkpoint_lsn,
+      // Handle legacy values
+      slot_name: doc.slot_name ?? `powersync_${doc._id}`,
+      last_fatal_error: doc.last_fatal_error,
+      last_fatal_error_ts: doc.last_fatal_error_ts,
+      last_checkpoint_ts: doc.last_checkpoint_ts,
+      last_keepalive_ts: doc.last_keepalive_ts,
+      active: doc.state == 'ACTIVE',
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION
+    });
   }
 
-  parsed(options: storage.ParseSyncRulesOptions) {
-    return new MongoPersistedSyncRules(
-      this.id,
-      SqlSyncRules.fromYaml(this.sync_rules_content, options),
-      this.last_checkpoint_lsn,
-      this.slot_name
-    );
+  getStorageConfig() {
+    const storageConfig = getMongoStorageConfig(this.storageVersion);
+    if (storageConfig == null) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
+      );
+    }
+    return storageConfig;
   }
 
   async lock() {

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -32,7 +32,14 @@ import * as timers from 'timers/promises';
 import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js';
+import {
+  BucketDataDocument,
+  BucketDataKey,
+  BucketStateDocument,
+  SourceKey,
+  SourceTableDocument,
+  StorageConfig
+} from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactor } from './MongoCompactor.js';
@@ -40,7 +47,8 @@ import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 
 export interface MongoSyncBucketStorageOptions {
-  checksumOptions?: MongoChecksumOptions;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  storageConfig: StorageConfig;
 }
 
 /**
@@ -69,12 +77,15 @@ export class MongoSyncBucketStorage
     public readonly group_id: number,
     private readonly sync_rules: storage.PersistedSyncRulesContent,
     public readonly slot_name: string,
-    writeCheckpointMode?: storage.WriteCheckpointMode,
-    options?: MongoSyncBucketStorageOptions
+    writeCheckpointMode: storage.WriteCheckpointMode | undefined,
+    options: MongoSyncBucketStorageOptions
   ) {
     super();
     this.db = factory.db;
-    this.checksums = new MongoChecksums(this.db, this.group_id, options?.checksumOptions);
+    this.checksums = new MongoChecksums(this.db, this.group_id, {
+      ...options.checksumOptions,
+      storageConfig: options?.storageConfig
+    });
     this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
       db: this.db,
       mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,

package/src/storage/implementation/models.ts

@@ -1,4 +1,4 @@
-import { InternalOpId, storage } from '@powersync/service-core';
+import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { event_types } from '@powersync/service-types';
@@ -199,11 +199,35 @@ export interface SyncRuleDocument {
   last_fatal_error_ts: Date | null;
 
   content: string;
+  serialized_plan?: SerializedSyncPlan | null;
 
   lock?: {
     id: string;
     expires_at: Date;
   } | null;
+
+  storage_version?: number;
+}
+
+export interface StorageConfig extends storage.StorageVersionConfig {
+  /**
+   * When true, bucket_data.checksum is guaranteed to be persisted as a Long.
+   *
+   * When false, it could also have been persisted as an Int32 or Double, in which case it must be converted to
+   * a Long before summing.
+   */
+  longChecksums: boolean;
+}
+
+const LONG_CHECKSUMS_STORAGE_VERSION = 2;
+
+export function getMongoStorageConfig(storageVersion: number): StorageConfig | undefined {
+  const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
+  if (baseConfig == null) {
+    return undefined;
+  }
+
+  return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION };
 }
 
 export interface CheckpointEventDocument {

package/src/storage/storage-index.ts

@@ -2,7 +2,6 @@ export * from './implementation/db.js';
 export * from './implementation/models.js';
 export * from './implementation/MongoBucketBatch.js';
 export * from './implementation/MongoIdSequence.js';
-export * from './implementation/MongoPersistedSyncRules.js';
 export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncBucketStorage.js';

package/src/utils/test-utils.ts

@@ -1,14 +1,13 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { PowerSyncMongo } from '../storage/implementation/db.js';
 import { TestStorageOptions } from '@powersync/service-core';
+import { MongoBucketStorage, MongoBucketStorageOptions } from '../storage/MongoBucketStorage.js';
 import { MongoReportStorage } from '../storage/MongoReportStorage.js';
-import { MongoBucketStorage } from '../storage/MongoBucketStorage.js';
-import { MongoSyncBucketStorageOptions } from '../storage/implementation/MongoSyncBucketStorage.js';
+import { PowerSyncMongo } from '../storage/implementation/db.js';
 
 export type MongoTestStorageOptions = {
   url: string;
   isCI: boolean;
-  internalOptions?: MongoSyncBucketStorageOptions;
+  internalOptions?: MongoBucketStorageOptions;
 };
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {