@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0

package/src/storage/implementation/MongoParameterCompactor.ts

@@ -2,8 +2,14 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { logger } from '@powersync/lib-services-framework';
 import { bson, CompactOptions, InternalOpId } from '@powersync/service-core';
 import { LRUCache } from 'lru-cache';
-import { VersionedPowerSyncMongo } from './db.js';
-import { BucketParameterDocument } from './models.js';
+import type { VersionedPowerSyncMongo } from './db.js';
+
+type ParameterCompactionReadDocument = {
+  _id: InternalOpId;
+  key: mongo.Document;
+  lookup: unknown;
+  bucket_parameters?: unknown[] | null;
+};
 
 /**
  * Compacts parameter lookup data (the bucket_parameters collection).
@@ -12,16 +18,28 @@ import { BucketParameterDocument } from './models.js';
  *
  * For background, see the `/docs/parameters-lookups.md` file.
  */
-export class MongoParameterCompactor {
+export abstract class MongoParameterCompactor {
   constructor(
-    private db: VersionedPowerSyncMongo,
-    private group_id: number,
-    private checkpoint: InternalOpId,
-    private options: CompactOptions
+    protected readonly db: VersionedPowerSyncMongo,
+    protected readonly group_id: number,
+    protected readonly checkpoint: InternalOpId,
+    protected readonly options: CompactOptions
   ) {}
 
   async compact() {
     logger.info(`Compacting parameters for sync config ${this.group_id} up to checkpoint ${this.checkpoint}`);
+    for (const collection of await this.getCollections()) {
+      await this.compactCollection(collection);
+    }
+  }
+
+  protected abstract getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+
+  protected abstract collectionFilter(): mongo.Document;
+
+  protected abstract deleteFilter(doc: mongo.Document): mongo.Document;
+
+  protected async compactCollection(collection: mongo.Collection<mongo.Document>) {
     // This is the currently-active checkpoint.
     // We do not remove any data that may be used by this checkpoint.
     // snapshot queries ensure that if any clients are still using older checkpoints, they would
@@ -32,43 +50,38 @@ export class MongoParameterCompactor {
     // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup)
     // in MongoDB already. However, that risks running into cases where MongoDB needs to process
     // very large amounts of data before returning results, which could lead to timeouts.
-    const cursor = this.db.bucket_parameters.find(
-      {
-        'key.g': this.group_id
-      },
-      {
-        sort: { lookup: 1, _id: 1 },
-        batchSize: 10_000,
-        projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 }
-      }
-    );
+    const cursor = collection.find(this.collectionFilter(), {
+      sort: { lookup: 1, _id: 1 },
+      batchSize: 10_000,
+      projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 }
+    });
 
     // The index doesn't cover sorting by key, so we keep our own cache of the last seen key.
     let lastByKey = new LRUCache<string, InternalOpId>({
       max: this.options.compactParameterCacheLimit ?? 10_000
     });
     let removeIds: InternalOpId[] = [];
-    let removeDeleted: mongo.AnyBulkWriteOperation<BucketParameterDocument>[] = [];
+    let removeDeleted: mongo.AnyBulkWriteOperation<mongo.Document>[] = [];
     let checkedEntries = 0;
     let checkedEntriesAtLastLog = 0;
     let lastProgressLogTime = Date.now();
 
     const flush = async (force: boolean) => {
       if (removeIds.length >= 1000 || (force && removeIds.length > 0)) {
-        const results = await this.db.bucket_parameters.deleteMany({ _id: { $in: removeIds } });
+        const results = await collection.deleteMany({ _id: { $in: removeIds } } as any);
         logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`);
         removeIds = [];
       }
 
       if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) {
-        const results = await this.db.bucket_parameters.bulkWrite(removeDeleted);
+        const results = await collection.bulkWrite(removeDeleted);
         logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`);
         removeDeleted = [];
       }
     };
 
     while (await cursor.hasNext()) {
-      const batch = cursor.readBufferedDocuments();
+      const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[];
       checkedEntries += batch.length;
       const now = Date.now();
       if (now - lastProgressLogTime >= 60_000) {
@@ -79,7 +92,7 @@ export class MongoParameterCompactor {
         checkedEntriesAtLastLog = checkedEntries;
       }
 
-      for (let doc of batch) {
+      for (const doc of batch) {
         if (doc._id >= checkpoint) {
           continue;
         }
@@ -103,7 +116,7 @@ export class MongoParameterCompactor {
           // in the cache due to cache size limits. So we need to explicitly remove all earlier operations.
           removeDeleted.push({
             deleteMany: {
-              filter: { 'key.g': doc.key.g, lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key }
+              filter: this.deleteFilter(doc)
             }
           });
         }
@@ -113,6 +126,6 @@ export class MongoParameterCompactor {
     }
 
     await flush(true);
-    logger.info('Parameter compaction completed');
+    logger.info(`Parameter compaction completed for ${collection.collectionName}`);
   }
 }

package/src/storage/implementation/MongoPersistedSyncRules.ts

@@ -0,0 +1,76 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import {
+  BucketDataScope,
+  BucketDataSource,
+  CompatibilityOption,
+  DEFAULT_HYDRATION_STATE,
+  HydratedSyncRules,
+  HydrationState,
+  ParameterIndexLookupCreator,
+  SyncConfigWithErrors,
+  versionedHydrationState
+} from '@powersync/service-sync-rules';
+import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
+import { StorageConfig } from './models.js';
+
+export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
+  public readonly hydrationState: HydrationState;
+
+  constructor(
+    public readonly id: number,
+    public readonly sync_rules: SyncConfigWithErrors,
+    public readonly slot_name: string,
+    private readonly mapping: BucketDefinitionMapping | null,
+    private readonly storageConfig: StorageConfig
+  ) {
+    if (this.storageConfig.incrementalReprocessing) {
+      if (this.mapping == null) {
+        throw new ServiceAssertionError(`mapping is required for v3 storage`);
+      }
+      this.hydrationState = new MongoHydrationState(this.mapping, this.id);
+    } else if (
+      !this.sync_rules.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) &&
+      !this.storageConfig.versionedBuckets
+    ) {
+      this.hydrationState = DEFAULT_HYDRATION_STATE;
+    } else {
+      this.hydrationState = versionedHydrationState(this.id);
+    }
+  }
+
+  hydratedSyncRules(): HydratedSyncRules {
+    return this.sync_rules.config.hydrate({ hydrationState: this.hydrationState });
+  }
+}
+
+class MongoHydrationState implements HydrationState {
+  constructor(
+    private readonly mapping: BucketDefinitionMapping,
+    private readonly version: number
+  ) {}
+
+  getBucketSourceScope(source: BucketDataSource): BucketDataScope {
+    // Keep this aligned with versionedHydrationState() for now.
+    //
+    // Previous Mongo-specific behavior:
+    // const defId = this.mapping.bucketSourceId(source);
+    // return {
+    //   bucketPrefix: defId,
+    //   source
+    // };
+    return {
+      bucketPrefix: `${this.version}#${source.uniqueName}`,
+      source
+    };
+  }
+
+  getParameterIndexLookupScope(source: ParameterIndexLookupCreator) {
+    const defId = this.mapping.parameterLookupId(source);
+    return {
+      lookupName: defId,
+      queryId: '',
+      source
+    };
+  }
+}

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -1,11 +1,14 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
+import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
+import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
-import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js';
+import { PowerSyncMongo } from './db.js';
 import { getMongoStorageConfig, SyncRuleDocument } from './models.js';
 
 export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
   public current_lock: MongoSyncRulesLock | null = null;
+  public readonly mapping: BucketDefinitionMapping;
 
   constructor(
     private db: PowerSyncMongo,
@@ -25,12 +28,26 @@ export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesCo
       active: doc.state == 'ACTIVE',
       storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION
     });
+    this.mapping = BucketDefinitionMapping.fromSyncRules(doc);
   }
 
   getStorageConfig() {
     return getMongoStorageConfig(this.storageVersion);
   }
 
+  parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules {
+    const parsed = super.parsed(options);
+    const storageConfig = this.getStorageConfig();
+
+    return new MongoPersistedSyncRules(
+      parsed.id,
+      parsed.sync_rules,
+      parsed.slot_name,
+      storageConfig.incrementalReprocessing ? this.mapping : null,
+      storageConfig
+    );
+  }
+
   async lock() {
     const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this);
     this.current_lock = lock;

package/src/storage/implementation/MongoStorageProvider.ts

@@ -3,8 +3,8 @@ import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersy
 import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
 import { MongoStorageConfig } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { PowerSyncMongo } from './db.js';
 import { MongoReportStorage } from '../MongoReportStorage.js';
+import { PowerSyncMongo } from './db.js';
 
 export class MongoStorageProvider implements storage.StorageProvider {
   get type() {