@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts

@@ -0,0 +1,80 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { DO_NOT_LOG } from '@powersync/lib-services-framework';
+import { PowerSyncMongo } from '../db.js';
+import { CommonSourceTableDocument, StorageConfig } from '../models.js';
+
+export abstract class BaseVersionedPowerSyncMongo {
+  readonly client: mongo.MongoClient;
+  readonly db: mongo.Db;
+  readonly storageConfig: StorageConfig;
+  [DO_NOT_LOG] = true;
+
+  constructor(
+    protected readonly upstream: PowerSyncMongo,
+    storageConfig: StorageConfig
+  ) {
+    this.client = upstream.client;
+    this.db = upstream.db;
+    this.storageConfig = storageConfig;
+  }
+
+  get bucket_data() {
+    return this.upstream.bucket_data;
+  }
+
+  get op_id_sequence() {
+    return this.upstream.op_id_sequence;
+  }
+
+  get sync_rules() {
+    return this.upstream.sync_rules;
+  }
+
+  get custom_write_checkpoints() {
+    return this.upstream.custom_write_checkpoints;
+  }
+
+  get write_checkpoints() {
+    return this.upstream.write_checkpoints;
+  }
+
+  get instance() {
+    return this.upstream.instance;
+  }
+
+  get locks() {
+    return this.upstream.locks;
+  }
+
+  get checkpoint_events() {
+    return this.upstream.checkpoint_events;
+  }
+
+  get connection_report_events() {
+    return this.upstream.connection_report_events;
+  }
+
+  notifyCheckpoint() {
+    return this.upstream.notifyCheckpoint();
+  }
+
+  protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
+    return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
+  }
+
+  protected sourceTableCollectionName(replicationStreamId: number) {
+    return this.upstream.sourceTableCollectionName(replicationStreamId);
+  }
+
+  protected async listCollectionsByPrefix<T extends mongo.Document>(prefix: string): Promise<mongo.Collection<T>[]> {
+    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<T>(collection.name));
+  }
+
+  abstract commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument>;
+
+  abstract initializeStreamStorage(replicationStreamId: number): Promise<void>;
+}

package/src/storage/implementation/createMongoSyncBucketStorage.ts

@@ -0,0 +1,25 @@
+import { storage } from '@powersync/service-core';
+import { MongoBucketStorage } from '../MongoBucketStorage.js';
+import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
+import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
+import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
+
+export { MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
+
+export type { MongoSyncBucketStorage };
+
+export function createMongoSyncBucketStorage(
+  factory: MongoBucketStorage,
+  group_id: number,
+  sync_rules: MongoPersistedSyncRulesContentV1,
+  slot_name: string,
+  writeCheckpointMode: storage.WriteCheckpointMode | undefined,
+  options: MongoSyncBucketStorageOptions
+): MongoSyncBucketStorage {
+  if (sync_rules.getStorageConfig().incrementalReprocessing) {
+    return new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  }
+
+  return new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+}

package/src/storage/implementation/db.ts

@@ -2,24 +2,30 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
 
-import { DO_NOT_LOG, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { DO_NOT_LOG } from '@powersync/lib-services-framework';
 import { MongoStorageConfig } from '../../types/types.js';
+import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBase.js';
 import {
-  BucketDataDocument,
-  BucketParameterDocument,
-  BucketStateDocument,
   CheckpointEventDocument,
   ClientConnectionDocument,
-  CurrentDataDocument,
-  CurrentDataDocumentV3,
+  CommonSourceTableDocument,
   CustomWriteCheckpointDocument,
   IdSequenceDocument,
   InstanceDocument,
   SourceTableDocument,
   StorageConfig,
-  SyncRuleDocument,
+  SyncRuleDocumentBase,
   WriteCheckpointDocument
 } from './models.js';
+import {
+  BucketDataDocumentV1,
+  BucketParameterDocument,
+  BucketStateDocumentV1,
+  CurrentDataDocument
+} from './v1/models.js';
+import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
+import { BucketDataDocumentV3 } from './v3/models.js';
+import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -32,17 +38,16 @@ export class PowerSyncMongo {
   [DO_NOT_LOG] = true;
 
   readonly current_data: mongo.Collection<CurrentDataDocument>;
-  readonly v3_current_data: mongo.Collection<CurrentDataDocumentV3>;
-  readonly bucket_data: mongo.Collection<BucketDataDocument>;
+  readonly bucket_data: mongo.Collection<BucketDataDocumentV1>;
   readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
   readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
-  readonly sync_rules: mongo.Collection<SyncRuleDocument>;
+  readonly sync_rules: mongo.Collection<SyncRuleDocumentBase>;
   readonly source_tables: mongo.Collection<SourceTableDocument>;
   readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
   readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
   readonly instance: mongo.Collection<InstanceDocument>;
   readonly locks: mongo.Collection<lib_mongo.locks.Lock>;
-  readonly bucket_state: mongo.Collection<BucketStateDocument>;
+  readonly bucket_state: mongo.Collection<BucketStateDocumentV1>;
   readonly checkpoint_events: mongo.Collection<CheckpointEventDocument>;
   readonly connection_report_events: mongo.Collection<ClientConnectionDocument>;
 
@@ -58,7 +63,6 @@ export class PowerSyncMongo {
     this.db = db;
 
     this.current_data = db.collection('current_data');
-    this.v3_current_data = db.collection('v3_current_data');
     this.bucket_data = db.collection('bucket_data');
     this.bucket_parameters = db.collection('bucket_parameters');
     this.op_id_sequence = db.collection('op_id_sequence');
@@ -73,8 +77,83 @@ export class PowerSyncMongo {
     this.connection_report_events = this.db.collection('connection_report_events');
   }
 
-  versioned(storageConfig: StorageConfig) {
-    return new VersionedPowerSyncMongo(this, storageConfig);
+  versioned(storageConfig: StorageConfig & { incrementalReprocessing: true }): VersionedPowerSyncMongoV3;
+  versioned(storageConfig: StorageConfig & { incrementalReprocessing: false }): VersionedPowerSyncMongoV1;
+  versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo;
+  versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo {
+    if (storageConfig.incrementalReprocessing) {
+      return new VersionedPowerSyncMongoV3(this, storageConfig);
+    }
+
+    return new VersionedPowerSyncMongoV1(this, storageConfig);
+  }
+
+  /**
+   * Not safe for user-provided prefix - only for hardcoded values.
+   */
+  async listBucketDataCollectionsV3(groupId?: number): Promise<mongo.Collection<BucketDataDocumentV3>[]> {
+    const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
+    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<BucketDataDocumentV3>(collection.name));
+  }
+
+  /**
+   * Not safe for user-provided prefix - only for hardcoded values.
+   */
+  private async collectionsByPrefix(prefix: string): Promise<mongo.Collection<never>[]> {
+    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<never>(collection.name));
+  }
+
+  /**
+   * List all parameter index collections across all replication streams.
+   *
+   * Primarily used to clear the db.
+   */
+  async listAllParameterIndexCollectionsV3(): Promise<mongo.Collection<never>[]> {
+    return this.collectionsByPrefix(`parameter_index_`);
+  }
+
+  /**
+   * List all parameter index collections across all replication streams.
+   *
+   * Primarily used to clear the db.
+   */
+  async listAllSourceRecordCollectionsV3(): Promise<mongo.Collection<never>[]> {
+    return this.collectionsByPrefix(`source_records_`);
+  }
+
+  async listAllBucketStateCollectionsV3(): Promise<mongo.Collection<never>[]> {
+    return this.collectionsByPrefix(`bucket_state_`);
+  }
+
+  sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
+    return `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`;
+  }
+
+  sourceTableCollectionName(replicationStreamId: number) {
+    return `source_table_${replicationStreamId}`;
+  }
+
+  async listSourceTableCollections(
+    replicationStreamId?: number
+  ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
+    const filter =
+      replicationStreamId == null
+        ? { name: new RegExp('^source_table_') }
+        : { name: this.sourceTableCollectionName(replicationStreamId) };
+    const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId);
+    const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<CommonSourceTableDocument>(collection.name));
   }
 
   /**
@@ -82,11 +161,25 @@ export class PowerSyncMongo {
    */
   async clear() {
     await this.current_data.deleteMany({});
-    await this.v3_current_data.deleteMany({});
+    for (const collection of await this.listAllSourceRecordCollectionsV3()) {
+      await collection.drop();
+    }
     await this.bucket_data.deleteMany({});
+    for (const collection of await this.listBucketDataCollectionsV3()) {
+      await collection.drop();
+    }
     await this.bucket_parameters.deleteMany({});
+    for (const collection of await this.listAllParameterIndexCollectionsV3()) {
+      await collection.drop();
+    }
+    for (const collection of await this.listAllBucketStateCollectionsV3()) {
+      await collection.drop();
+    }
     await this.op_id_sequence.deleteMany({});
     await this.sync_rules.deleteMany({});
+    for (const collection of await this.listSourceTableCollections()) {
+      await collection.drop();
+    }
     await this.source_tables.deleteMany({});
     await this.write_checkpoints.deleteMany({});
     await this.instance.deleteOne({});
@@ -105,7 +198,7 @@ export class PowerSyncMongo {
   }
 
   /**
-   * Call this after every checkpoint or sync rules status update. Rather call too often than too rarely.
+   * Call this after every checkpoint or replication stream status update. Rather call too often than too rarely.
    *
    * This is used in a similar way to the Postgres NOTIFY functionality.
    */
@@ -183,126 +276,12 @@ export class PowerSyncMongo {
       { name: 'dirty_count' }
     );
   }
-
-  async initializeStorageVersion(storageConfig: StorageConfig) {
-    if (storageConfig.softDeleteCurrentData) {
-      // Initialize the v3_current_data collection, which is used for the new storage version.
-      // No-op if this already exists
-      await this.v3_current_data.createIndex(
-        {
-          '_id.g': 1,
-          pending_delete: 1
-        },
-        {
-          partialFilterExpression: { pending_delete: { $exists: true } },
-          name: 'pending_delete'
-        }
-      );
-    }
-  }
 }
 
 /**
  * This is similar to PowerSyncMongo, but blocks access to certain collections based on the storage version.
  */
-export class VersionedPowerSyncMongo {
-  readonly client: mongo.MongoClient;
-  readonly db: mongo.Db;
-  [DO_NOT_LOG] = true;
-
-  readonly storageConfig: StorageConfig;
-  #upstream: PowerSyncMongo;
-
-  constructor(upstream: PowerSyncMongo, storageConfig: StorageConfig) {
-    this.#upstream = upstream;
-    this.client = upstream.client;
-    this.db = upstream.db;
-    this.storageConfig = storageConfig;
-  }
-
-  /**
-   * Uses either `current_data` or `v3_current_data` collection based on the storage version.
-   *
-   * Use in places where it does not matter which version is used.
-   */
-  get common_current_data(): mongo.Collection<CurrentDataDocument> {
-    if (this.storageConfig.softDeleteCurrentData) {
-      return this.#upstream.v3_current_data;
-    } else {
-      return this.#upstream.current_data;
-    }
-  }
-
-  get v1_current_data() {
-    if (this.storageConfig.softDeleteCurrentData) {
-      throw new ServiceAssertionError(
-        'current_data collection should not be used when softDeleteCurrentData is enabled'
-      );
-    }
-    return this.#upstream.current_data;
-  }
-
-  get v3_current_data() {
-    if (!this.storageConfig.softDeleteCurrentData) {
-      throw new ServiceAssertionError(
-        'v3_current_data collection should not be used when softDeleteCurrentData is disabled'
-      );
-    }
-    return this.#upstream.v3_current_data;
-  }
-
-  get bucket_data() {
-    return this.#upstream.bucket_data;
-  }
-
-  get bucket_parameters() {
-    return this.#upstream.bucket_parameters;
-  }
-
-  get op_id_sequence() {
-    return this.#upstream.op_id_sequence;
-  }
-
-  get sync_rules() {
-    return this.#upstream.sync_rules;
-  }
-
-  get source_tables() {
-    return this.#upstream.source_tables;
-  }
-
-  get custom_write_checkpoints() {
-    return this.#upstream.custom_write_checkpoints;
-  }
-
-  get write_checkpoints() {
-    return this.#upstream.write_checkpoints;
-  }
-
-  get instance() {
-    return this.#upstream.instance;
-  }
-
-  get locks() {
-    return this.#upstream.locks;
-  }
-
-  get bucket_state() {
-    return this.#upstream.bucket_state;
-  }
-
-  get checkpoint_events() {
-    return this.#upstream.checkpoint_events;
-  }
-
-  get connection_report_events() {
-    return this.#upstream.connection_report_events;
-  }
-
-  notifyCheckpoint() {
-    return this.#upstream.notifyCheckpoint();
-  }
-}
+export type VersionedPowerSyncMongo = BaseVersionedPowerSyncMongo;
 
 export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) {
   return new PowerSyncMongo(