@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

package/src/storage/implementation/MongoParameterCompactor.ts

@@ -2,8 +2,14 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { logger } from '@powersync/lib-services-framework';
 import { bson, CompactOptions, InternalOpId } from '@powersync/service-core';
 import { LRUCache } from 'lru-cache';
-import { VersionedPowerSyncMongo } from './db.js';
-import { BucketParameterDocument } from './models.js';
+import type { VersionedPowerSyncMongo } from './db.js';
+
+type ParameterCompactionReadDocument = {
+  _id: InternalOpId;
+  key: mongo.Document;
+  lookup: unknown;
+  bucket_parameters?: unknown[] | null;
+};
 
 /**
  * Compacts parameter lookup data (the bucket_parameters collection).
@@ -12,16 +18,28 @@ import { BucketParameterDocument } from './models.js';
  *
  * For background, see the `/docs/parameters-lookups.md` file.
  */
-export class MongoParameterCompactor {
+export abstract class MongoParameterCompactor {
   constructor(
-    private db: VersionedPowerSyncMongo,
-    private group_id: number,
-    private checkpoint: InternalOpId,
-    private options: CompactOptions
+    protected readonly db: VersionedPowerSyncMongo,
+    protected readonly group_id: number,
+    protected readonly checkpoint: InternalOpId,
+    protected readonly options: CompactOptions
   ) {}
 
   async compact() {
     logger.info(`Compacting parameters for sync config ${this.group_id} up to checkpoint ${this.checkpoint}`);
+    for (const collection of await this.getCollections()) {
+      await this.compactCollection(collection);
+    }
+  }
+
+  protected abstract getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+
+  protected abstract collectionFilter(): mongo.Document;
+
+  protected abstract deleteFilter(doc: mongo.Document): mongo.Document;
+
+  protected async compactCollection(collection: mongo.Collection<mongo.Document>) {
     // This is the currently-active checkpoint.
     // We do not remove any data that may be used by this checkpoint.
     // snapshot queries ensure that if any clients are still using older checkpoints, they would
@@ -32,43 +50,38 @@ export class MongoParameterCompactor {
     // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup)
     // in MongoDB already. However, that risks running into cases where MongoDB needs to process
     // very large amounts of data before returning results, which could lead to timeouts.
-    const cursor = this.db.bucket_parameters.find(
-      {
-        'key.g': this.group_id
-      },
-      {
-        sort: { lookup: 1, _id: 1 },
-        batchSize: 10_000,
-        projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 }
-      }
-    );
+    const cursor = collection.find(this.collectionFilter(), {
+      sort: { lookup: 1, _id: 1 },
+      batchSize: 10_000,
+      projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 }
+    });
 
     // The index doesn't cover sorting by key, so we keep our own cache of the last seen key.
     let lastByKey = new LRUCache<string, InternalOpId>({
       max: this.options.compactParameterCacheLimit ?? 10_000
     });
     let removeIds: InternalOpId[] = [];
-    let removeDeleted: mongo.AnyBulkWriteOperation<BucketParameterDocument>[] = [];
+    let removeDeleted: mongo.AnyBulkWriteOperation<mongo.Document>[] = [];
     let checkedEntries = 0;
     let checkedEntriesAtLastLog = 0;
     let lastProgressLogTime = Date.now();
 
     const flush = async (force: boolean) => {
       if (removeIds.length >= 1000 || (force && removeIds.length > 0)) {
-        const results = await this.db.bucket_parameters.deleteMany({ _id: { $in: removeIds } });
+        const results = await collection.deleteMany({ _id: { $in: removeIds } } as any);
         logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`);
         removeIds = [];
       }
 
       if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) {
-        const results = await this.db.bucket_parameters.bulkWrite(removeDeleted);
+        const results = await collection.bulkWrite(removeDeleted);
         logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`);
         removeDeleted = [];
       }
     };
 
     while (await cursor.hasNext()) {
-      const batch = cursor.readBufferedDocuments();
+      const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[];
       checkedEntries += batch.length;
       const now = Date.now();
       if (now - lastProgressLogTime >= 60_000) {
@@ -79,7 +92,7 @@ export class MongoParameterCompactor {
         checkedEntriesAtLastLog = checkedEntries;
       }
 
-      for (let doc of batch) {
+      for (const doc of batch) {
         if (doc._id >= checkpoint) {
           continue;
         }
@@ -103,7 +116,7 @@ export class MongoParameterCompactor {
           // in the cache due to cache size limits. So we need to explicitly remove all earlier operations.
           removeDeleted.push({
             deleteMany: {
-              filter: { 'key.g': doc.key.g, lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key }
+              filter: this.deleteFilter(doc)
             }
           });
         }
@@ -113,6 +126,6 @@ export class MongoParameterCompactor {
     }
 
     await flush(true);
-    logger.info('Parameter compaction completed');
+    logger.info(`Parameter compaction completed for ${collection.collectionName}`);
   }
 }

package/src/storage/implementation/MongoPersistedSyncRules.ts

@@ -0,0 +1,82 @@
+import * as sqlite from 'node:sqlite';
+
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import {
+  BucketDataScope,
+  BucketDataSource,
+  CompatibilityOption,
+  DEFAULT_HYDRATION_STATE,
+  HydratedSyncConfig,
+  HydrationState,
+  nodeSqlite,
+  ParameterIndexLookupCreator,
+  ParameterLookupScope,
+  SyncConfigWithErrors,
+  versionedHydrationState
+} from '@powersync/service-sync-rules';
+import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
+import { StorageConfig } from './models.js';
+
+export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
+  public readonly hydrationState: HydrationState;
+
+  constructor(
+    public readonly id: number,
+    public readonly syncConfigWithErrors: SyncConfigWithErrors,
+    public readonly slot_name: string,
+    private readonly mapping: BucketDefinitionMapping | null,
+    private readonly storageConfig: StorageConfig
+  ) {
+    if (this.storageConfig.incrementalReprocessing) {
+      if (this.mapping == null) {
+        throw new ServiceAssertionError(`mapping is required for v3 storage`);
+      }
+      this.hydrationState = new MongoHydrationState(this.mapping, this.id);
+    } else if (
+      !this.syncConfigWithErrors.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) &&
+      !this.storageConfig.versionedBuckets
+    ) {
+      this.hydrationState = DEFAULT_HYDRATION_STATE;
+    } else {
+      this.hydrationState = versionedHydrationState(this.id);
+    }
+  }
+
+  hydratedSyncConfig(): HydratedSyncConfig {
+    return this.syncConfigWithErrors.config.hydrate({
+      hydrationState: this.hydrationState,
+      sqlite: nodeSqlite(sqlite)
+    });
+  }
+}
+
+class MongoHydrationState implements HydrationState {
+  constructor(
+    private readonly mapping: BucketDefinitionMapping,
+    private readonly version: number
+  ) {}
+
+  getBucketSourceScope(source: BucketDataSource): BucketDataScope {
+    // Keep this aligned with versionedHydrationState() for now.
+    //
+    // Previous Mongo-specific behavior:
+    // return {
+    //   bucketPrefix: defId,
+    //   source
+    // };
+    return {
+      bucketPrefix: `${this.version}#${source.uniqueName}`,
+      source
+    };
+  }
+
+  getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope {
+    const defId = this.mapping.parameterLookupId(source);
+    return {
+      lookupName: defId,
+      queryId: '',
+      source
+    };
+  }
+}

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -1,17 +1,60 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { storage } from '@powersync/service-core';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { storage, SyncRuleState } from '@powersync/service-core';
+import * as bson from 'bson';
+import { ReplicationStreamDocumentV3, SyncConfigDefinition } from '../storage-index.js';
+import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
+import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
 import { PowerSyncMongo } from './db.js';
-import { getMongoStorageConfig, SyncRuleDocument } from './models.js';
+import { getMongoStorageConfig } from './models.js';
+import { SyncRuleDocumentV1 } from './v1/models.js';
 
-export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
+abstract class MongoPersistedSyncRulesContentBase extends storage.PersistedSyncRulesContent {
   public current_lock: MongoSyncRulesLock | null = null;
+  public readonly mapping: BucketDefinitionMapping;
+  public readonly syncConfigId: bson.ObjectId | null;
 
-  constructor(
-    private db: PowerSyncMongo,
-    doc: mongo.WithId<SyncRuleDocument>
+  protected constructor(
+    protected readonly db: PowerSyncMongo,
+    options: ConstructorParameters<typeof storage.PersistedSyncRulesContent>[0] & {
+      mapping: BucketDefinitionMapping;
+      syncConfigId: bson.ObjectId | null;
+    }
   ) {
-    super({
+    const { mapping, syncConfigId, ...base } = options;
+    super(base);
+    this.mapping = mapping;
+    this.syncConfigId = syncConfigId;
+  }
+
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
+  }
+
+  parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules {
+    const parsed = super.parsed(options);
+    const storageConfig = this.getStorageConfig();
+
+    return new MongoPersistedSyncRules(
+      parsed.id,
+      parsed.syncConfigWithErrors,
+      parsed.slot_name,
+      storageConfig.incrementalReprocessing ? this.mapping : null,
+      storageConfig
+    );
+  }
+
+  async lock(session?: mongo.ClientSession) {
+    const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this, session);
+    this.current_lock = lock;
+    return lock;
+  }
+}
+
+export class MongoPersistedSyncRulesContentV1 extends MongoPersistedSyncRulesContentBase {
+  constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1) {
+    super(db, {
       id: doc._id,
       sync_rules_content: doc.content,
       compiled_plan: doc.serialized_plan ?? null,
@@ -22,18 +65,37 @@ export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesCo
       last_fatal_error_ts: doc.last_fatal_error_ts,
       last_checkpoint_ts: doc.last_checkpoint_ts,
       last_keepalive_ts: doc.last_keepalive_ts,
-      active: doc.state == 'ACTIVE',
-      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION
+      active: doc.state == SyncRuleState.ACTIVE,
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
+      mapping: new BucketDefinitionMapping(),
+      syncConfigId: null
     });
   }
+}
 
-  getStorageConfig() {
-    return getMongoStorageConfig(this.storageVersion);
-  }
+export class MongoPersistedSyncRulesContentV3 extends MongoPersistedSyncRulesContentBase {
+  declare public readonly syncConfigId: bson.ObjectId;
 
-  async lock() {
-    const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this);
-    this.current_lock = lock;
-    return lock;
+  constructor(db: PowerSyncMongo, doc: ReplicationStreamDocumentV3, config: SyncConfigDefinition) {
+    const state = doc.sync_configs.find((c) => c._id.equals(config._id));
+    if (state == null) {
+      throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`);
+    }
+    super(db, {
+      id: doc._id,
+      sync_rules_content: config.content,
+      compiled_plan: config.serialized_plan ?? null,
+
+      last_checkpoint_lsn: state?.last_checkpoint_lsn ?? null,
+      slot_name: doc.slot_name ?? `powersync_${doc._id}`,
+      last_fatal_error: doc.last_fatal_error,
+      last_fatal_error_ts: doc.last_fatal_error_ts,
+      last_checkpoint_ts: doc.last_checkpoint_ts,
+      last_keepalive_ts: doc.last_keepalive_ts,
+      active: doc.state == SyncRuleState.ACTIVE && state.state == SyncRuleState.ACTIVE,
+      storageVersion: doc.storage_version,
+      mapping: BucketDefinitionMapping.fromSyncConfig(config),
+      syncConfigId: config._id
+    });
   }
 }