@powersync/service-module-mongodb-storage 0.17.0 → 0.18.1

package/src/storage/implementation/MongoParameterCompactor.ts

@@ -18,12 +18,13 @@ type ParameterCompactionReadDocument = {
  *
  * For background, see the `/docs/parameters-lookups.md` file.
  */
-export abstract class MongoParameterCompactor {
+export class MongoParameterCompactor {
   constructor(
     protected readonly db: VersionedPowerSyncMongo,
     protected readonly group_id: number,
     protected readonly checkpoint: InternalOpId,
-    protected readonly options: CompactOptions
+    protected readonly options: CompactOptions,
+    protected readonly getCollectionsCb?: () => Promise<mongo.Collection<mongo.Document>[]>
   ) {}
 
   async compact() {
@@ -33,11 +34,27 @@ export abstract class MongoParameterCompactor {
     }
   }
 
-  protected abstract getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+  protected async getCollections(): Promise<mongo.Collection<mongo.Document>[]> {
+    if (this.getCollectionsCb == null) {
+      throw new Error('getCollections callback not provided');
+    }
+    const collections = await this.getCollectionsCb();
+    // Cast from the version-specific collection type to the generic Document type
+    // used by the parameter compactor base class.
+    return collections.map((collection) => collection as unknown as mongo.Collection<mongo.Document>);
+  }
 
-  protected abstract collectionFilter(): mongo.Document;
+  protected collectionFilter(): mongo.Document {
+    return {};
+  }
 
-  protected abstract deleteFilter(doc: mongo.Document): mongo.Document;
+  protected deleteFilter(doc: mongo.Document): mongo.Document {
+    return {
+      lookup: doc.lookup,
+      _id: { $lte: doc._id },
+      key: doc.key
+    };
+  }
 
   protected async compactCollection(collection: mongo.Collection<mongo.Document>) {
     // This is the currently-active checkpoint.
@@ -68,6 +85,7 @@ export abstract class MongoParameterCompactor {
 
     const flush = async (force: boolean) => {
       if (removeIds.length >= 1000 || (force && removeIds.length > 0)) {
+        // MongoDB Filter<T> doesn't fully match our dynamic delete filter shape here.
         const results = await collection.deleteMany({ _id: { $in: removeIds } } as any);
         logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`);
         removeIds = [];
@@ -81,6 +99,7 @@ export abstract class MongoParameterCompactor {
     };
 
     while (await cursor.hasNext()) {
+      // readBufferedDocuments returns a generic type; we know the shape from our projection.
       const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[];
       checkedEntries += batch.length;
       const now = Date.now();

package/src/storage/implementation/MongoParsedSyncConfigSet.ts

@@ -0,0 +1,88 @@
+import * as sqlite from 'node:sqlite';
+
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  BucketDefinitionMapping,
+  MultiSyncConfigBucketDefinitionMapping,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage,
+  SyncConfigWithMapping,
+  SyncConfigWithRequiredMapping
+} from '@powersync/service-core';
+import {
+  CompatibilityOption,
+  DEFAULT_HYDRATION_STATE,
+  HydratedSyncConfig,
+  HydrationState,
+  nodeSqlite,
+  SyncConfigWithErrors,
+  versionedHydrationState
+} from '@powersync/service-sync-rules';
+import { StorageConfig } from './models.js';
+import { MongoHydrationState } from './MongoHydrationState.js';
+
+export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet {
+  public readonly hydrationState: HydrationState;
+  public readonly syncConfigs: SyncConfigWithErrors[];
+  public readonly replicationStreamName: string;
+  public readonly mapping: BucketDefinitionMapping;
+
+  constructor(
+    public readonly replicationStreamId: number,
+    storageConfig: StorageConfig,
+    slotName: string,
+    syncConfigs: SyncConfigWithMapping[]
+  ) {
+    this.replicationStreamName = slotName;
+    this.syncConfigs = syncConfigs.map((config) => config.syncConfig);
+    if (this.syncConfigs.length == 0) {
+      throw new ServiceAssertionError(`At least one sync config is required`);
+    }
+    const [firstConfig] = this.syncConfigs;
+    const compatibility = firstConfig.config.compatibility;
+    for (const config of this.syncConfigs) {
+      if (config.config.compatibility.equals(compatibility)) {
+        continue;
+      }
+      throw new ServiceAssertionError(
+        `All sync configs in a replication stream must use the same compatibility options`
+      );
+    }
+
+    if (storageConfig.incrementalReprocessing) {
+      if (syncConfigs.some((c) => c.mapping == null)) {
+        throw new ServiceAssertionError(`mapping is required for v3 storage`);
+      }
+      const mappedConfigs = syncConfigs as SyncConfigWithRequiredMapping[];
+      this.hydrationState = new MongoHydrationState(mappedConfigs, this.replicationStreamId);
+      this.mapping = new MultiSyncConfigBucketDefinitionMapping(mappedConfigs);
+    } else if (!compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && !storageConfig.versionedBuckets) {
+      const [syncConfig] = syncConfigs;
+      if (syncConfigs.length != 1 || syncConfig == null) {
+        throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
+      }
+      this.hydrationState = DEFAULT_HYDRATION_STATE;
+      this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
+    } else {
+      const [syncConfig] = syncConfigs;
+      if (syncConfigs.length != 1 || syncConfig == null) {
+        throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
+      }
+      this.hydrationState = versionedHydrationState(this.replicationStreamId);
+      this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
+    }
+  }
+
+  #hydratedSyncConfig: HydratedSyncConfig | undefined;
+
+  get hydratedSyncConfig(): HydratedSyncConfig {
+    this.#hydratedSyncConfig ??= new HydratedSyncConfig({
+      definitions: this.syncConfigs.map((config) => config.config),
+      createParams: {
+        hydrationState: this.hydrationState,
+        sqlite: nodeSqlite(sqlite)
+      }
+    });
+    return this.#hydratedSyncConfig;
+  }
+}

package/src/storage/implementation/MongoPersistedReplicationStream.ts

@@ -0,0 +1,118 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { ReplicationStreamStorageIds, SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { ReplicationStreamDocumentV3, SyncConfigDefinition } from '../storage-index.js';
+import { PowerSyncMongo } from './db.js';
+import { getMongoStorageConfig } from './models.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+import {
+  MongoPersistedSyncConfigContentBase,
+  MongoPersistedSyncConfigContentV1,
+  MongoPersistedSyncConfigContentV3
+} from './MongoPersistedSyncConfigContent.js';
+import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
+import { SyncRuleDocumentV1 } from './v1/models.js';
+
+export class MongoPersistedReplicationStream extends storage.PersistedReplicationStream {
+  public current_lock: MongoSyncRulesLock | null = null;
+  public readonly syncConfigContent: readonly MongoPersistedSyncConfigContentBase[];
+
+  constructor(
+    private readonly db: PowerSyncMongo,
+    private readonly doc: SyncRuleDocumentV1 | ReplicationStreamDocumentV3,
+    private readonly configs: SyncConfigDefinition[] = []
+  ) {
+    const storageVersion = doc.storage_version ?? storage.LEGACY_STORAGE_VERSION;
+    const replicationJobId =
+      configs.length == 0
+        ? String(doc._id)
+        : `${doc._id}:${configs
+            .map((config) => config._id.toHexString())
+            .sort()
+            .join(',')}`;
+
+    super({
+      replicationStreamId: doc._id,
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      state: doc.state,
+      storageVersion,
+      replicationJobId
+    });
+
+    this.syncConfigContent = this.createSyncConfigContent();
+  }
+
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
+  }
+
+  private createSyncConfigContent(): MongoPersistedSyncConfigContentBase[] {
+    if (this.getStorageConfig().incrementalReprocessing) {
+      if (this.configs.length == 0) {
+        throw new ServiceAssertionError(`Cannot create v3 storage without sync config definitions`);
+      }
+      return this.configs.map(
+        (config) => new MongoPersistedSyncConfigContentV3(this.db, this.doc as ReplicationStreamDocumentV3, config)
+      );
+    }
+
+    return [new MongoPersistedSyncConfigContentV1(this.db, this.doc as SyncRuleDocumentV1)];
+  }
+
+  get syncConfigIds(): bson.ObjectId[] {
+    return this.configs.map((config) => config._id);
+  }
+
+  #storageIds: ReplicationStreamStorageIds | undefined;
+
+  /**
+   * Persisted storage ids for all sync configs in this replication stream.
+   *
+   * Derived from the persisted rule_mapping documents - requires no parsed sync configs.
+   */
+  get storageIds(): ReplicationStreamStorageIds {
+    this.#storageIds ??= new ReplicationStreamStorageIds(this.syncConfigContent.map((content) => content.mapping));
+    return this.#storageIds;
+  }
+
+  /**
+   * Parse the sync configs for this replication stream.
+   *
+   * This creates a fresh parse on every call. Do not call this from operational paths -
+   * use {@link MongoSyncBucketStorage.getParsedSyncConfigSet} instead, which returns a
+   * canonical instance so that parsed source objects and mappings stay associated.
+   */
+  parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
+    const storageConfig = this.getStorageConfig();
+    if (!storageConfig.incrementalReprocessing) {
+      return this.syncConfigContent[0].parsed(options);
+    }
+
+    const syncConfigs = this.configs.map((config) => {
+      return {
+        syncConfigId: config._id.toHexString(),
+        syncConfig: storage.parsePersistedSyncConfigContent({
+          content: config.content,
+          compiledPlan: config.serialized_plan ?? null,
+          storageVersion: this.storageVersion,
+          parseOptions: options
+        }),
+        mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping)
+      };
+    });
+
+    return new MongoParsedSyncConfigSet(
+      this.replicationStreamId,
+      storageConfig,
+      this.replicationStreamName,
+      syncConfigs
+    );
+  }
+
+  async lock(session?: mongo.ClientSession) {
+    const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this, session);
+    this.current_lock = lock;
+    return lock;
+  }
+}

package/src/storage/implementation/MongoPersistedSyncConfigContent.ts

@@ -0,0 +1,154 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import {
+  ReplicationStreamDocumentV3,
+  SyncConfigDefinition,
+  SyncRuleConfigStateV3,
+  SyncRuleDocumentV1
+} from '../storage-index.js';
+import { PowerSyncMongo } from './db.js';
+import { getMongoStorageConfig } from './models.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+
+export abstract class MongoPersistedSyncConfigContentBase extends storage.PersistedSyncConfigContent {
+  public readonly mapping: SingleSyncConfigBucketDefinitionMapping;
+  public readonly syncConfigObjectId: bson.ObjectId | null;
+
+  protected constructor(
+    protected readonly db: PowerSyncMongo,
+    options: Omit<storage.PersistedSyncConfigContentData, 'syncConfigId'> & {
+      mapping: SingleSyncConfigBucketDefinitionMapping;
+      syncConfigId: bson.ObjectId | null;
+    }
+  ) {
+    const { mapping, syncConfigId, ...base } = options;
+    super({
+      ...base,
+      syncConfigId: syncConfigId?.toHexString() ?? null
+    });
+    this.mapping = mapping;
+    this.syncConfigObjectId = syncConfigId;
+  }
+
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
+  }
+
+  parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
+    const parsed = super.parsed(options);
+    const storageConfig = this.getStorageConfig();
+    const [syncConfig] = parsed.syncConfigs;
+    if (syncConfig == null) {
+      throw new ServiceAssertionError(`Expected one parsed sync config`);
+    }
+
+    if (storageConfig.incrementalReprocessing) {
+      const syncConfigId = this.syncConfigId;
+      if (syncConfigId == null) {
+        throw new ServiceAssertionError(`syncConfigId is required for v3 sync config content`);
+      }
+
+      return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
+        { syncConfigId, syncConfig, mapping: this.mapping }
+      ]);
+    }
+
+    return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
+      { syncConfig, mapping: null }
+    ]);
+  }
+}
+export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase {
+  constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1) {
+    super(db, {
+      replicationStreamId: doc._id,
+      sync_rules_content: doc.content,
+      compiled_plan: doc.serialized_plan ?? null,
+      // Handle legacy values
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
+      mapping: new SingleSyncConfigBucketDefinitionMapping(),
+      syncConfigId: null,
+      syncConfigState: doc.state
+    });
+  }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    const doc = await this.db.sync_rules.findOne<SyncRuleDocumentV1>({ _id: this.replicationStreamId });
+    if (doc == null) {
+      return null;
+    }
+
+    return syncConfigStatusFromV1(doc);
+  }
+}
+export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase {
+  declare public readonly syncConfigObjectId: bson.ObjectId;
+
+  constructor(db: PowerSyncMongo, doc: ReplicationStreamDocumentV3, config: SyncConfigDefinition) {
+    const state = doc.sync_configs.find((c) => c._id.equals(config._id));
+    if (state == null) {
+      throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`);
+    }
+    super(db, {
+      replicationStreamId: doc._id,
+      sync_rules_content: config.content,
+      compiled_plan: config.serialized_plan ?? null,
+
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      storageVersion: doc.storage_version,
+      mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
+      syncConfigId: config._id,
+      syncConfigState: state.state
+    });
+  }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    const doc = await this.db.sync_rules.findOne<ReplicationStreamDocumentV3>({
+      _id: this.replicationStreamId,
+      'sync_configs._id': this.syncConfigObjectId
+    });
+    if (doc == null) {
+      return null;
+    }
+
+    const state = doc.sync_configs.find((c) => c._id.equals(this.syncConfigObjectId));
+    if (state == null) {
+      return null;
+    }
+
+    return syncConfigStatusFromV3(doc, state);
+  }
+}
+
+function syncConfigStatusFromV1(doc: SyncRuleDocumentV1): storage.PersistedSyncConfigStatus {
+  return {
+    id: String(doc._id),
+    replicationStreamId: doc._id,
+    state: doc.state,
+    snapshot_done: doc.snapshot_done,
+    last_checkpoint_lsn: doc.last_checkpoint_lsn,
+    last_fatal_error: doc.last_fatal_error,
+    last_fatal_error_ts: doc.last_fatal_error_ts,
+    last_keepalive_ts: doc.last_keepalive_ts,
+    last_checkpoint_ts: doc.last_checkpoint_ts
+  };
+}
+
+function syncConfigStatusFromV3(
+  doc: ReplicationStreamDocumentV3,
+  state: SyncRuleConfigStateV3
+): storage.PersistedSyncConfigStatus {
+  return {
+    id: state._id.toHexString(),
+    replicationStreamId: doc._id,
+    state: state.state,
+    snapshot_done: state.snapshot_done,
+    last_checkpoint_lsn: state.last_checkpoint_lsn,
+    last_fatal_error: doc.last_fatal_error,
+    last_fatal_error_ts: doc.last_fatal_error_ts,
+    last_keepalive_ts: doc.last_keepalive_ts,
+    last_checkpoint_ts: doc.last_checkpoint_ts
+  };
+}

package/src/storage/implementation/MongoStorageProvider.ts

@@ -39,8 +39,11 @@ export class MongoStorageProvider implements storage.StorageProvider {
 
     const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
     const syncStorageFactory = new MongoBucketStorage(database, {
-      // TODO currently need the entire resolved config due to this
-      slot_name_prefix: resolvedConfig.slot_name_prefix
+      replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
+      // Right now, only MongoDB source databases supports incremental reprocessing.
+      // Remove this filter when we support it for other source databases.
+      // This assumes a single source connection - revisit if we ever support multiple connections.
+      supportsMultipleSyncConfigs: resolvedConfig.connections?.[0]?.type == lib_mongo.MONGO_CONNECTION_TYPE
     });
 
     // Storage factory for reports