@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/src/storage/implementation/MongoParsedSyncConfigSet.ts

@@ -0,0 +1,122 @@
+import * as sqlite from 'node:sqlite';
+
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  BucketDefinitionMapping,
+  MultiSyncConfigBucketDefinitionMapping,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage,
+  SyncConfigWithMapping,
+  SyncConfigWithRequiredMapping
+} from '@powersync/service-core';
+import {
+  CompatibilityOption,
+  DEFAULT_HYDRATION_STATE,
+  EventDefinitionId,
+  HydratedEventDescriptor,
+  HydratedSyncConfig,
+  HydrationState,
+  nodeSqlite,
+  SyncConfigWithErrors,
+  versionedHydrationState
+} from '@powersync/service-sync-rules';
+import { StorageConfig } from './models.js';
+import { MongoHydrationState } from './MongoHydrationState.js';
+
+export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet {
+  public readonly hydrationState: HydrationState;
+  public readonly syncConfigs: SyncConfigWithErrors[];
+  public readonly replicationStreamName: string;
+  public readonly mapping: BucketDefinitionMapping;
+
+  readonly #configsWithMapping: readonly SyncConfigWithMapping[];
+
+  constructor(
+    public readonly replicationStreamId: number,
+    storageConfig: StorageConfig,
+    slotName: string,
+    syncConfigs: SyncConfigWithMapping[]
+  ) {
+    this.replicationStreamName = slotName;
+    this.#configsWithMapping = [...syncConfigs];
+    this.syncConfigs = syncConfigs.map((config) => config.syncConfig);
+    if (this.syncConfigs.length == 0) {
+      throw new ServiceAssertionError(`At least one sync config is required`);
+    }
+    const [firstConfig] = this.syncConfigs;
+    const compatibility = firstConfig.config.compatibility;
+    for (const config of this.syncConfigs) {
+      if (config.config.compatibility.equals(compatibility)) {
+        continue;
+      }
+      throw new ServiceAssertionError(
+        `All sync configs in a replication stream must use the same compatibility options`
+      );
+    }
+
+    if (storageConfig.incrementalReprocessing) {
+      if (syncConfigs.some((c) => c.mapping == null)) {
+        throw new ServiceAssertionError(`mapping is required for v3 storage`);
+      }
+      const mappedConfigs = syncConfigs as SyncConfigWithRequiredMapping[];
+      this.hydrationState = new MongoHydrationState(mappedConfigs, this.replicationStreamId);
+      this.mapping = new MultiSyncConfigBucketDefinitionMapping(mappedConfigs);
+    } else if (!compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && !storageConfig.versionedBuckets) {
+      const [syncConfig] = syncConfigs;
+      if (syncConfigs.length != 1 || syncConfig == null) {
+        throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
+      }
+      this.hydrationState = DEFAULT_HYDRATION_STATE;
+      this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
+    } else {
+      const [syncConfig] = syncConfigs;
+      if (syncConfigs.length != 1 || syncConfig == null) {
+        throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
+      }
+      this.hydrationState = versionedHydrationState(this.replicationStreamId);
+      this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
+    }
+  }
+
+  #hydratedSyncConfig: HydratedSyncConfig | undefined;
+
+  get hydratedSyncConfig(): HydratedSyncConfig {
+    this.#hydratedSyncConfig ??= new HydratedSyncConfig({
+      definitions: this.syncConfigs.map((config) => config.config),
+      createParams: {
+        hydrationState: this.hydrationState,
+        sqlite: nodeSqlite(sqlite)
+      }
+    });
+    return this.#hydratedSyncConfig;
+  }
+
+  #eventById: ReadonlyMap<EventDefinitionId, HydratedEventDescriptor> | undefined;
+
+  /**
+   * Hydrated events for the replication stream, keyed by their assigned storage id.
+   *
+   * Each config's events are resolved against that config's own (single-config) mapping, so the resolution is
+   * unambiguous, and the result is deduplicated by assigned id: unchanged events shared across configs collapse to
+   * one entry, while a changed event keeps a separate entry under its new id.
+   */
+  get eventById(): ReadonlyMap<EventDefinitionId, HydratedEventDescriptor> {
+    if (this.#eventById == null) {
+      const map = new Map<EventDefinitionId, HydratedEventDescriptor>();
+      const byDefinition = this.hydratedSyncConfig.eventDescriptorsByDefinition;
+      for (const config of this.#configsWithMapping) {
+        if (config.mapping == null) {
+          continue;
+        }
+        for (const event of byDefinition.get(config.syncConfig.config) ?? []) {
+          const id = config.mapping.eventId(event);
+          if (!map.has(id)) {
+            map.set(id, event);
+          }
+        }
+      }
+      this.#eventById = map;
+    }
+    return this.#eventById;
+  }
+}

package/src/storage/implementation/MongoPersistedReplicationStream.ts

@@ -0,0 +1,118 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { ReplicationStreamStorageIds, SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { ReplicationStreamDocumentV3, SyncConfigDefinition } from '../storage-index.js';
+import { PowerSyncMongo } from './db.js';
+import { getMongoStorageConfig } from './models.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+import {
+  MongoPersistedSyncConfigContentBase,
+  MongoPersistedSyncConfigContentV1,
+  MongoPersistedSyncConfigContentV3
+} from './MongoPersistedSyncConfigContent.js';
+import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
+import { SyncRuleDocumentV1 } from './v1/models.js';
+
+export class MongoPersistedReplicationStream extends storage.PersistedReplicationStream {
+  public current_lock: MongoSyncRulesLock | null = null;
+  public readonly syncConfigContent: readonly MongoPersistedSyncConfigContentBase[];
+
+  constructor(
+    private readonly db: PowerSyncMongo,
+    private readonly doc: SyncRuleDocumentV1 | ReplicationStreamDocumentV3,
+    private readonly configs: SyncConfigDefinition[] = []
+  ) {
+    const storageVersion = doc.storage_version ?? storage.LEGACY_STORAGE_VERSION;
+    const replicationJobId =
+      configs.length == 0
+        ? String(doc._id)
+        : `${doc._id}:${configs
+            .map((config) => config._id.toHexString())
+            .sort()
+            .join(',')}`;
+
+    super({
+      replicationStreamId: doc._id,
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      state: doc.state,
+      storageVersion,
+      replicationJobId
+    });
+
+    this.syncConfigContent = this.createSyncConfigContent();
+  }
+
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
+  }
+
+  private createSyncConfigContent(): MongoPersistedSyncConfigContentBase[] {
+    if (this.getStorageConfig().incrementalReprocessing) {
+      if (this.configs.length == 0) {
+        throw new ServiceAssertionError(`Cannot create v3 storage without sync config definitions`);
+      }
+      return this.configs.map(
+        (config) => new MongoPersistedSyncConfigContentV3(this.db, this.doc as ReplicationStreamDocumentV3, config)
+      );
+    }
+
+    return [new MongoPersistedSyncConfigContentV1(this.db, this.doc as SyncRuleDocumentV1)];
+  }
+
+  get syncConfigIds(): bson.ObjectId[] {
+    return this.configs.map((config) => config._id);
+  }
+
+  #storageIds: ReplicationStreamStorageIds | undefined;
+
+  /**
+   * Persisted storage ids for all sync configs in this replication stream.
+   *
+   * Derived from the persisted rule_mapping documents - requires no parsed sync configs.
+   */
+  get storageIds(): ReplicationStreamStorageIds {
+    this.#storageIds ??= new ReplicationStreamStorageIds(this.syncConfigContent.map((content) => content.mapping));
+    return this.#storageIds;
+  }
+
+  /**
+   * Parse the sync configs for this replication stream.
+   *
+   * This creates a fresh parse on every call. Do not call this from operational paths -
+   * use {@link MongoSyncBucketStorage.getParsedSyncConfigSet} instead, which returns a
+   * canonical instance so that parsed source objects and mappings stay associated.
+   */
+  parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
+    const storageConfig = this.getStorageConfig();
+    if (!storageConfig.incrementalReprocessing) {
+      return this.syncConfigContent[0].parsed(options);
+    }
+
+    const syncConfigs = this.configs.map((config) => {
+      return {
+        syncConfigId: config._id.toHexString(),
+        syncConfig: storage.parsePersistedSyncConfigContent({
+          content: config.content,
+          compiledPlan: config.serialized_plan ?? null,
+          storageVersion: this.storageVersion,
+          parseOptions: options
+        }),
+        mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping)
+      };
+    });
+
+    return new MongoParsedSyncConfigSet(
+      this.replicationStreamId,
+      storageConfig,
+      this.replicationStreamName,
+      syncConfigs
+    );
+  }
+
+  async lock(session?: mongo.ClientSession) {
+    const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this, session);
+    this.current_lock = lock;
+    return lock;
+  }
+}

package/src/storage/implementation/MongoPersistedSyncConfigContent.ts

@@ -0,0 +1,156 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import {
+  ReplicationStreamDocumentV3,
+  SyncConfigDefinition,
+  SyncRuleConfigStateV3,
+  SyncRuleDocumentV1
+} from '../storage-index.js';
+import { PowerSyncMongo } from './db.js';
+import { getMongoStorageConfig } from './models.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+
+export abstract class MongoPersistedSyncConfigContentBase extends storage.PersistedSyncConfigContent {
+  public readonly mapping: SingleSyncConfigBucketDefinitionMapping;
+  public readonly syncConfigObjectId: bson.ObjectId | null;
+
+  protected constructor(
+    protected readonly db: PowerSyncMongo,
+    options: Omit<storage.PersistedSyncConfigContentData, 'syncConfigId'> & {
+      mapping: SingleSyncConfigBucketDefinitionMapping;
+      syncConfigId: bson.ObjectId | null;
+    }
+  ) {
+    const { mapping, syncConfigId, ...base } = options;
+    super({
+      ...base,
+      syncConfigId: syncConfigId?.toHexString() ?? null
+    });
+    this.mapping = mapping;
+    this.syncConfigObjectId = syncConfigId;
+  }
+
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
+  }
+
+  parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
+    const parsed = super.parsed(options);
+    const storageConfig = this.getStorageConfig();
+    const [syncConfig] = parsed.syncConfigs;
+    if (syncConfig == null) {
+      throw new ServiceAssertionError(`Expected one parsed sync config`);
+    }
+
+    if (storageConfig.incrementalReprocessing) {
+      const syncConfigId = this.syncConfigId;
+      if (syncConfigId == null) {
+        throw new ServiceAssertionError(`syncConfigId is required for v3 sync config content`);
+      }
+
+      return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
+        { syncConfigId, syncConfig, mapping: this.mapping }
+      ]);
+    }
+
+    return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
+      { syncConfig, mapping: null }
+    ]);
+  }
+}
+export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase {
+  constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1) {
+    super(db, {
+      replicationStreamId: doc._id,
+      sync_rules_content: doc.content,
+      compiled_plan: doc.serialized_plan ?? null,
+      // Handle legacy values
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
+      mapping: new SingleSyncConfigBucketDefinitionMapping(),
+      syncConfigId: null,
+      syncConfigState: doc.state,
+      version_label: doc.version_label ?? undefined
+    });
+  }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    const doc = await this.db.sync_rules.findOne<SyncRuleDocumentV1>({ _id: this.replicationStreamId });
+    if (doc == null) {
+      return null;
+    }
+
+    return syncConfigStatusFromV1(doc);
+  }
+}
+export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase {
+  declare public readonly syncConfigObjectId: bson.ObjectId;
+
+  constructor(db: PowerSyncMongo, doc: ReplicationStreamDocumentV3, config: SyncConfigDefinition) {
+    const state = doc.sync_configs.find((c) => c._id.equals(config._id));
+    if (state == null) {
+      throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`);
+    }
+    super(db, {
+      replicationStreamId: doc._id,
+      sync_rules_content: config.content,
+      compiled_plan: config.serialized_plan ?? null,
+
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      storageVersion: doc.storage_version,
+      mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
+      syncConfigId: config._id,
+      syncConfigState: state.state,
+      version_label: config.version_label
+    });
+  }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    const doc = await this.db.sync_rules.findOne<ReplicationStreamDocumentV3>({
+      _id: this.replicationStreamId,
+      'sync_configs._id': this.syncConfigObjectId
+    });
+    if (doc == null) {
+      return null;
+    }
+
+    const state = doc.sync_configs.find((c) => c._id.equals(this.syncConfigObjectId));
+    if (state == null) {
+      return null;
+    }
+
+    return syncConfigStatusFromV3(doc, state);
+  }
+}
+
+function syncConfigStatusFromV1(doc: SyncRuleDocumentV1): storage.PersistedSyncConfigStatus {
+  return {
+    id: String(doc._id),
+    replicationStreamId: doc._id,
+    state: doc.state,
+    snapshot_done: doc.snapshot_done,
+    last_checkpoint_lsn: doc.last_checkpoint_lsn,
+    last_fatal_error: doc.last_fatal_error,
+    last_fatal_error_ts: doc.last_fatal_error_ts,
+    last_keepalive_ts: doc.last_keepalive_ts,
+    last_checkpoint_ts: doc.last_checkpoint_ts
+  };
+}
+
+function syncConfigStatusFromV3(
+  doc: ReplicationStreamDocumentV3,
+  state: SyncRuleConfigStateV3
+): storage.PersistedSyncConfigStatus {
+  return {
+    id: state._id.toHexString(),
+    replicationStreamId: doc._id,
+    state: state.state,
+    snapshot_done: state.snapshot_done,
+    last_checkpoint_lsn: state.last_checkpoint_lsn,
+    last_fatal_error: doc.last_fatal_error,
+    last_fatal_error_ts: doc.last_fatal_error_ts,
+    last_keepalive_ts: doc.last_keepalive_ts,
+    last_checkpoint_ts: doc.last_checkpoint_ts
+  };
+}

package/src/storage/implementation/MongoStorageProvider.ts

@@ -1,10 +1,11 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
 import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
-import { MongoStorageConfig } from '../../types/types.js';
+import { MongoStorageConfig, normalizeClearBatchThrottleRate } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { MongoReportStorage } from '../MongoReportStorage.js';
 import { PowerSyncMongo } from './db.js';
+import type { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
 
 export class MongoStorageProvider implements storage.StorageProvider {
   get type() {
@@ -23,6 +24,24 @@ export class MongoStorageProvider implements storage.StorageProvider {
     }
 
     const decodedConfig = MongoStorageConfig.decode(storage as any);
+
+    let objectStorage: ObjectStorage | undefined;
+    if (decodedConfig.object_storage?.type === 's3') {
+      // Dynamically import S3ObjectStorage to avoid loading AWS SDK unless needed.
+      const { S3ObjectStorage } = await import('./v3/object-storage/S3ObjectStorage.js');
+      objectStorage = new S3ObjectStorage({
+        bucket: decodedConfig.object_storage.bucket,
+        region: decodedConfig.object_storage.region,
+        prefix: decodedConfig.object_storage.prefix,
+        endpoint: decodedConfig.object_storage.endpoint,
+        forcePathStyle: decodedConfig.object_storage.force_path_style,
+        accessKeyId: decodedConfig.object_storage.access_key_id,
+        secretAccessKey: decodedConfig.object_storage.secret_access_key,
+        concurrencyLimit: decodedConfig.object_storage.concurrency_limit,
+        defaultsMode: decodedConfig.object_storage.defaults_mode
+      });
+    }
+
     const client = lib_mongo.db.createMongoClient(decodedConfig, {
       powersyncVersion: POWERSYNC_VERSION,
       maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
@@ -38,9 +57,27 @@ export class MongoStorageProvider implements storage.StorageProvider {
     await client.connect();
 
     const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
+    const readPreference =
+      decodedConfig.bulk_read_preference == null
+        ? undefined
+        : new lib_mongo.mongo.ReadPreference(decodedConfig.bulk_read_preference, undefined, {
+            // maxStalenessSeconds is relevant for all modes except 'primary'.
+            // 90 is the minimum value.
+            maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
+          });
     const syncStorageFactory = new MongoBucketStorage(database, {
-      // TODO currently need the entire resolved config due to this
-      slot_name_prefix: resolvedConfig.slot_name_prefix
+      replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
+      readPreference,
+      clearBatchThrottleRate: normalizeClearBatchThrottleRate(decodedConfig.clear_batch_throttle_rate),
+      checksumCacheTtlMs: resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
+      defaultStorageVersion: decodedConfig.default_storage_version,
+      // Right now, only MongoDB source databases supports incremental reprocessing.
+      // Remove this filter when we support it for other source databases.
+      // This assumes a single source connection - revisit if we ever support multiple connections.
+      supportsMultipleSyncConfigs: resolvedConfig.connections?.[0]?.type == lib_mongo.MONGO_CONNECTION_TYPE,
+
+      objectStorage,
+      inlineThresholdBytes: decodedConfig.object_storage?.inline_threshold_bytes
     });
 
     // Storage factory for reports
@@ -53,8 +90,15 @@ export class MongoStorageProvider implements storage.StorageProvider {
         await syncStorageFactory[Symbol.asyncDispose]();
         await client.close();
       },
-      tearDown: () => {
+      tearDown: async () => {
         logger.info(`Tearing down storage: ${database.db.namespace}...`);
+        if (objectStorage != null) {
+          logger.info(
+            `Clearing object storage: ${decodedConfig.object_storage!.bucket}/${decodedConfig.object_storage!.prefix ?? ''}.`
+          );
+          const { objectCount } = await objectStorage.deletePrefix('bucket-data/');
+          logger.info(`Deleted ${objectCount} objects from object storage.`);
+        }
         return database.db.dropDatabase();
       },
       onFatalError: (callback) => {