@powersync/service-module-mongodb-storage 0.13.2 → 0.14.0

package/src/storage/implementation/MongoCompactor.ts

@@ -63,6 +63,7 @@ const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 const DEFAULT_MIN_BUCKET_CHANGES = 10;
 const DEFAULT_MIN_CHANGE_RATIO = 0.1;
+const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
 
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
@@ -538,31 +539,60 @@ export class MongoCompactor {
     let lastId = { g: this.group_id, b: new mongo.MinKey() as any };
     const maxId = { g: this.group_id, b: new mongo.MaxKey() as any };
     while (true) {
-      const batch = await this.db.bucket_state
-        .find(
-          {
-            _id: { $gt: lastId, $lt: maxId },
-            'estimate_since_compact.count': { $gte: options.minBucketChanges }
-          },
-          {
-            projection: {
-              _id: 1,
-              estimate_since_compact: 1,
-              compacted_state: 1
+      // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, we use an aggregation pipeline
+      // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria, rather than limiting
+      // on the output number.
+      const [result] = await this.db.bucket_state
+        .aggregate<{
+          buckets: Pick<BucketStateDocument, '_id' | 'estimate_since_compact' | 'compacted_state'>[];
+          cursor: Pick<BucketStateDocument, '_id'>[];
+        }>(
+          [
+            {
+              $match: {
+                _id: { $gt: lastId, $lt: maxId }
+              }
             },
-            sort: {
-              _id: 1
+            {
+              $sort: { _id: 1 }
             },
-            limit: 2000,
-            maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
-          }
+            {
+              // Scan a fixed number of docs each query so sparse matches don't block progress.
+              $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE
+            },
+            {
+              $facet: {
+                // This is the results for the batch
+                buckets: [
+                  {
+                    $match: {
+                      'estimate_since_compact.count': { $gte: options.minBucketChanges }
+                    }
+                  },
+                  {
+                    $project: {
+                      _id: 1,
+                      estimate_since_compact: 1,
+                      compacted_state: 1
+                    }
+                  }
+                ],
+                // This is used for the next query.
+                cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
+              }
+            }
+          ],
+          { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS }
         )
         .toArray();
-      if (batch.length == 0) {
+
+      const cursor = result?.cursor?.[0];
+      if (cursor == null) {
         break;
       }
-      lastId = batch[batch.length - 1]._id;
-      const mapped = batch.map((b) => {
+      lastId = cursor._id;
+
+      const mapped = (result?.buckets ?? []).map((b) => {
         const updatedCount = b.estimate_since_compact?.count ?? 0;
         const totalCount = (b.compacted_state?.count ?? 0) + updatedCount;
         const updatedBytes = b.estimate_since_compact?.bytes ?? 0;

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -1,48 +1,42 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
 import { PowerSyncMongo } from './db.js';
-import { SyncRuleDocument } from './models.js';
-
-export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRulesContent {
-  public readonly slot_name: string;
-
-  public readonly id: number;
-  public readonly sync_rules_content: string;
-  public readonly last_checkpoint_lsn: string | null;
-  public readonly last_fatal_error: string | null;
-  public readonly last_fatal_error_ts: Date | null;
-  public readonly last_keepalive_ts: Date | null;
-  public readonly last_checkpoint_ts: Date | null;
-  public readonly active: boolean;
+import { getMongoStorageConfig, SyncRuleDocument } from './models.js';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 
+export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
   public current_lock: MongoSyncRulesLock | null = null;
 
   constructor(
     private db: PowerSyncMongo,
     doc: mongo.WithId<SyncRuleDocument>
   ) {
-    this.id = doc._id;
-    this.sync_rules_content = doc.content;
-    this.last_checkpoint_lsn = doc.last_checkpoint_lsn;
-    // Handle legacy values
-    this.slot_name = doc.slot_name ?? `powersync_${this.id}`;
-    this.last_fatal_error = doc.last_fatal_error;
-    this.last_fatal_error_ts = doc.last_fatal_error_ts;
-    this.last_checkpoint_ts = doc.last_checkpoint_ts;
-    this.last_keepalive_ts = doc.last_keepalive_ts;
-    this.active = doc.state == 'ACTIVE';
+    super({
+      id: doc._id,
+      sync_rules_content: doc.content,
+      compiled_plan: doc.serialized_plan ?? null,
+      last_checkpoint_lsn: doc.last_checkpoint_lsn,
+      // Handle legacy values
+      slot_name: doc.slot_name ?? `powersync_${doc._id}`,
+      last_fatal_error: doc.last_fatal_error,
+      last_fatal_error_ts: doc.last_fatal_error_ts,
+      last_checkpoint_ts: doc.last_checkpoint_ts,
+      last_keepalive_ts: doc.last_keepalive_ts,
+      active: doc.state == 'ACTIVE',
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION
+    });
   }
 
-  parsed(options: storage.ParseSyncRulesOptions) {
-    return new MongoPersistedSyncRules(
-      this.id,
-      SqlSyncRules.fromYaml(this.sync_rules_content, options),
-      this.last_checkpoint_lsn,
-      this.slot_name
-    );
+  getStorageConfig() {
+    const storageConfig = getMongoStorageConfig(this.storageVersion);
+    if (storageConfig == null) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
+      );
+    }
+    return storageConfig;
   }
 
   async lock() {

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -32,7 +32,14 @@ import * as timers from 'timers/promises';
 import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js';
+import {
+  BucketDataDocument,
+  BucketDataKey,
+  BucketStateDocument,
+  SourceKey,
+  SourceTableDocument,
+  StorageConfig
+} from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactor } from './MongoCompactor.js';
@@ -40,7 +47,8 @@ import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 
 export interface MongoSyncBucketStorageOptions {
-  checksumOptions?: MongoChecksumOptions;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  storageConfig: StorageConfig;
 }
 
 /**
@@ -69,12 +77,15 @@ export class MongoSyncBucketStorage
     public readonly group_id: number,
     private readonly sync_rules: storage.PersistedSyncRulesContent,
     public readonly slot_name: string,
-    writeCheckpointMode?: storage.WriteCheckpointMode,
-    options?: MongoSyncBucketStorageOptions
+    writeCheckpointMode: storage.WriteCheckpointMode | undefined,
+    options: MongoSyncBucketStorageOptions
   ) {
     super();
     this.db = factory.db;
-    this.checksums = new MongoChecksums(this.db, this.group_id, options?.checksumOptions);
+    this.checksums = new MongoChecksums(this.db, this.group_id, {
+      ...options.checksumOptions,
+      storageConfig: options?.storageConfig
+    });
     this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
       db: this.db,
       mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,

package/src/storage/implementation/models.ts

@@ -1,4 +1,4 @@
-import { InternalOpId, storage } from '@powersync/service-core';
+import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { event_types } from '@powersync/service-types';
@@ -199,11 +199,35 @@ export interface SyncRuleDocument {
   last_fatal_error_ts: Date | null;
 
   content: string;
+  serialized_plan?: SerializedSyncPlan | null;
 
   lock?: {
     id: string;
     expires_at: Date;
   } | null;
+
+  storage_version?: number;
+}
+
+export interface StorageConfig extends storage.StorageVersionConfig {
+  /**
+   * When true, bucket_data.checksum is guaranteed to be persisted as a Long.
+   *
+   * When false, it could also have been persisted as an Int32 or Double, in which case it must be converted to
+   * a Long before summing.
+   */
+  longChecksums: boolean;
+}
+
+const LONG_CHECKSUMS_STORAGE_VERSION = 2;
+
+export function getMongoStorageConfig(storageVersion: number): StorageConfig | undefined {
+  const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
+  if (baseConfig == null) {
+    return undefined;
+  }
+
+  return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION };
 }
 
 export interface CheckpointEventDocument {

package/src/storage/storage-index.ts

@@ -2,7 +2,6 @@ export * from './implementation/db.js';
 export * from './implementation/models.js';
 export * from './implementation/MongoBucketBatch.js';
 export * from './implementation/MongoIdSequence.js';
-export * from './implementation/MongoPersistedSyncRules.js';
 export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncBucketStorage.js';

package/src/utils/test-utils.ts

@@ -1,14 +1,13 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { PowerSyncMongo } from '../storage/implementation/db.js';
 import { TestStorageOptions } from '@powersync/service-core';
+import { MongoBucketStorage, MongoBucketStorageOptions } from '../storage/MongoBucketStorage.js';
 import { MongoReportStorage } from '../storage/MongoReportStorage.js';
-import { MongoBucketStorage } from '../storage/MongoBucketStorage.js';
-import { MongoSyncBucketStorageOptions } from '../storage/implementation/MongoSyncBucketStorage.js';
+import { PowerSyncMongo } from '../storage/implementation/db.js';
 
 export type MongoTestStorageOptions = {
   url: string;
   isCI: boolean;
-  internalOptions?: MongoSyncBucketStorageOptions;
+  internalOptions?: MongoBucketStorageOptions;
 };
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {