@powersync/service-module-mongodb-storage 0.16.0 → 0.17.0

package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts

@@ -1,10 +1,12 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import {
   CheckpointChanges,
   GetCheckpointChangesOptions,
   InternalOpId,
   internalToExternalOpId,
+  maxLsn,
   ParameterSetLimitExceededError,
   ProtocolOpId,
   storage,
@@ -19,14 +21,22 @@ import {
   MongoSyncBucketStorageCheckpoint,
   MongoSyncBucketStorageContext
 } from '../common/MongoSyncBucketStorageContext.js';
-import { CommonSourceTableDocument } from '../models.js';
 import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
 import { MongoChecksums } from '../MongoChecksums.js';
 import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
-import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js';
+import {
+  MongoPersistedSyncRulesContentV1,
+  MongoPersistedSyncRulesContentV3
+} from '../MongoPersistedSyncRulesContent.js';
 import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
-import { BucketDataDocumentV3, BucketParameterDocumentV3, loadBucketDataDocumentV3 } from './models.js';
+import {
+  BucketDataDocumentV3,
+  BucketParameterDocumentV3,
+  loadBucketDataDocumentV3,
+  ReplicationStreamDocumentV3,
+  SyncRuleConfigStateV3
+} from './models.js';
 import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
 import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
 import { MongoCompactorV3 } from './MongoCompactorV3.js';
@@ -39,15 +49,67 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
   declare readonly db: VersionedPowerSyncMongoV3;
   declare readonly checksums: MongoChecksumsV3;
 
+  private readonly syncRulesV3: MongoPersistedSyncRulesContentV3;
+
   constructor(
     factory: MongoBucketStorage,
     group_id: number,
-    sync_rules: MongoPersistedSyncRulesContent,
+    sync_rules: MongoPersistedSyncRulesContentV1,
     slot_name: string,
     writeCheckpointMode: storage.WriteCheckpointMode | undefined,
     options: MongoSyncBucketStorageOptions
   ) {
     super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+    if (!(sync_rules instanceof MongoPersistedSyncRulesContentV3)) {
+      throw new ServiceAssertionError('Missing sync config id for storage v3');
+    }
+    this.syncRulesV3 = sync_rules;
+  }
+
+  private get syncConfigId(): bson.ObjectId {
+    return this.syncRulesV3.syncConfigId;
+  }
+
+  private get syncRulesCollection(): mongo.Collection<ReplicationStreamDocumentV3> {
+    return this.db.sync_rules as unknown as mongo.Collection<ReplicationStreamDocumentV3>;
+  }
+
+  private syncConfigMatch(extra: mongo.Document = {}): mongo.Filter<ReplicationStreamDocumentV3> {
+    return {
+      _id: this.group_id,
+      sync_configs: {
+        $elemMatch: {
+          _id: this.syncConfigId,
+          ...extra
+        }
+      }
+    };
+  }
+
+  private syncConfigProjection(extra: mongo.Document = {}): mongo.Document {
+    return {
+      ...extra,
+      sync_configs: {
+        $elemMatch: {
+          _id: this.syncConfigId
+        }
+      }
+    };
+  }
+
+  private syncConfigArrayFilters(): mongo.UpdateOptions['arrayFilters'] {
+    return [{ 'config._id': this.syncConfigId }];
+  }
+
+  /**
+   * For now, we only support a single sync config per replication stream.
+   *
+   * In the future we'll add support for multiple.
+   */
+  private selectedSyncConfig(
+    doc: Pick<ReplicationStreamDocumentV3, 'sync_configs'> | null
+  ): SyncRuleConfigStateV3 | null {
+    return doc?.sync_configs?.[0] ?? null;
   }
 
   protected async initializeVersionStorage(): Promise<void> {
@@ -100,30 +162,96 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     return new MongoBucketBatchV3(batchOptions);
   }
 
-  protected sourceTableBaseId(): Partial<CommonSourceTableDocument> {
-    return {};
+  protected async fetchCheckpointState(
+    session: mongo.ClientSession
+  ): Promise<{ checkpoint: bigint; lsn: string | null } | null> {
+    const doc = await this.syncRulesCollection.findOne(
+      this.syncConfigMatch({
+        state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
+      }),
+      {
+        session,
+        projection: this.syncConfigProjection()
+      }
+    );
+    const syncConfig = this.selectedSyncConfig(doc);
+    if (!syncConfig?.snapshot_done) {
+      return null;
+    }
+    return {
+      checkpoint: syncConfig.last_checkpoint ?? 0n,
+      lsn: syncConfig.last_checkpoint_lsn ?? null
+    };
+  }
+
+  protected async getWriterSyncState() {
+    const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
+      projection: this.syncConfigProjection({ snapshot_lsn: 1 })
+    });
+    const syncConfig = this.selectedSyncConfig(doc);
+    const checkpointLsn = syncConfig?.last_checkpoint_lsn ?? null;
+    return {
+      lastCheckpointLsn: checkpointLsn,
+      resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
+      keepaliveOp: syncConfig?.keepalive_op ?? null,
+      syncConfigId: this.syncConfigId
+    };
   }
 
-  protected augmentCreatedSourceTableDocument(
-    createDoc: CommonSourceTableDocument,
-    options: storage.ResolveTableOptions,
-    candidateSourceTable: storage.SourceTable
-  ): void {
-    const bucketDataSourceIds = options.sync_rules.definition.bucketDataSources
-      .filter((source) => source.tableSyncsData(candidateSourceTable))
-      .map((source) => this.mapping.bucketSourceId(source));
-    const parameterLookupSourceIds = options.sync_rules.definition.bucketParameterLookupSources
-      .filter((source) => source.tableSyncsParameters(candidateSourceTable))
-      .map((source) => this.mapping.parameterLookupId(source));
+  protected async terminateSyncRuleState(): Promise<void> {
+    await this.db.sync_rules.updateOne(
+      {
+        _id: this.group_id
+      },
+      {
+        $set: {
+          state: storage.SyncRuleState.TERMINATED,
+          persisted_lsn: null,
+          sync_configs: []
+        }
+      }
+    );
+  }
 
-    Object.assign(createDoc, {
-      bucket_data_source_ids: bucketDataSourceIds,
-      parameter_lookup_source_ids: parameterLookupSourceIds
+  protected async getStatusImpl(): Promise<storage.SyncRuleStatus> {
+    const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
+      projection: this.syncConfigProjection({ state: 1, snapshot_lsn: 1, keepalive_op: 1 })
     });
+    const syncConfig = this.selectedSyncConfig(doc);
+    if (doc == null || syncConfig == null) {
+      throw new ServiceAssertionError('Cannot find replication stream status');
+    }
+
+    return {
+      snapshot_done: syncConfig.snapshot_done ?? false,
+      snapshot_lsn: doc.snapshot_lsn ?? null,
+      active: doc.state == storage.SyncRuleState.ACTIVE && syncConfig.state == storage.SyncRuleState.ACTIVE,
+      checkpoint_lsn: syncConfig.last_checkpoint_lsn ?? null,
+      keepalive_op: syncConfig.keepalive_op ?? null
+    };
   }
 
-  protected async initializeResolvedSourceRecords(sourceTableId: bson.ObjectId): Promise<void> {
-    await this.db.initializeSourceRecordsCollection(this.group_id, sourceTableId);
+  protected async clearSyncRuleState(): Promise<void> {
+    await this.syncRulesCollection.updateOne(
+      this.syncConfigMatch(),
+      {
+        $set: {
+          persisted_lsn: null,
+          'sync_configs.$[config].snapshot_done': false,
+          'sync_configs.$[config].last_checkpoint_lsn': null,
+          'sync_configs.$[config].last_checkpoint': null,
+          'sync_configs.$[config].no_checkpoint_before': null,
+          'sync_configs.$[config].keepalive_op': null
+        },
+        $unset: {
+          snapshot_lsn: 1
+        }
+      },
+      {
+        maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+        arrayFilters: this.syncConfigArrayFilters()
+      }
+    );
   }
 
   protected override get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3> {

package/src/storage/implementation/v3/PersistedBatchV3.ts

@@ -1,10 +1,9 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { InternalOpId, storage } from '@powersync/service-core';
-import { BucketDataSource } from '@powersync/service-sync-rules';
+import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { mongoTableId } from '../../../utils/util.js';
-import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
 import {
   BucketStateUpdate,
   PersistedBatch,

package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts

@@ -1,5 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
+import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
 import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
 import { CommonSourceTableDocument } from '../models.js';
 import {
@@ -7,10 +7,15 @@ import {
   BucketParameterDocumentV3,
   BucketStateDocumentV3,
   CurrentDataDocumentV3,
-  SourceTableDocumentV3
+  SourceTableDocumentV3,
+  SyncConfigDefinition
 } from './models.js';
 
 export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
+  get syncConfigDefinitions() {
+    return this.db.collection<SyncConfigDefinition>('sync_config');
+  }
+
   sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
     const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
     return this.db.collection<CurrentDataDocumentV3>(collectionName);

package/src/storage/implementation/v3/models.ts

@@ -1,6 +1,6 @@
-import { InternalOpId } from '@powersync/service-core';
+import { InternalOpId, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
+import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
 import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
 import {
   BucketDataDocumentBase,
@@ -11,9 +11,77 @@ import {
   ReplicaId,
   SourceTableDocument,
   SourceTableKey,
+  SyncRuleCheckpointFields,
+  SyncRuleDocumentBase,
   TaggedBucketParameterDocument
 } from '../models.js';
 
+/**
+ * Embedded in sync_rules.sync_configs.
+ */
+export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields<bigint | null> {
+  _id: bson.ObjectId;
+
+  /**
+   * If false, we cannot create any checkpoints.
+   */
+  snapshot_done: boolean;
+
+  state: SyncRuleState;
+}
+
+/**
+ * Represents the state of a replication stream, in the sync_rules collection.
+ *
+ * Differences from V1:
+ * 1. The static config is moved into a separate sync_configs collection.
+ * 2. The same replication stream may be shared by multiple sync config instances.
+ */
+export interface ReplicationStreamDocumentV3 extends SyncRuleDocumentBase {
+  storage_version: number;
+
+  /**
+   * These contain the checkpoint/state per sync config.
+   *
+   * In common cases we'd have one active config or one active + one processing config,
+   * but the model allows multiple configs in any state.
+   */
+  sync_configs: SyncRuleConfigStateV3[];
+}
+
+/**
+ * Static sync config definition.
+ *
+ * This should be treated as immutable - we don't update this after initial creation.
+ */
+export interface SyncConfigDefinition {
+  _id: bson.ObjectId;
+  created_at: Date;
+  storage_version: number;
+  /**
+   * The related SyncRuleDocumentV3.
+   *
+   * Note that a specific sync config definition never moves between replication streams. Instead, we can create a new copy for the new replication stream.
+   *
+   * When terminating a specific sync config definition, we remove the reference from replication stream -> sync config, but this reference here remains as a historical record.
+   */
+  replication_stream_id: number;
+
+  content: string;
+  serialized_plan?: SerializedSyncPlan | null;
+
+  rule_mapping: {
+    /**
+     * Map of uniqueName -> id, unique per replication stream.
+     */
+    definitions: Record<string, string>;
+    /**
+     * Map of (lookupName, queryId) -> id, unique per replication stream.
+     */
+    parameter_indexes: Record<string, string>;
+  };
+}
+
 export interface CurrentBucketV3 extends CurrentBucket {
   def: BucketDefinitionId;
 }