@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts

@@ -1,11 +1,13 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import {
   CheckpointChanges,
   deserializeParameterLookup,
   GetCheckpointChangesOptions,
   InternalOpId,
   internalToExternalOpId,
+  maxLsn,
   ParameterSetLimitExceededError,
   ProtocolOpId,
   storage,
@@ -16,51 +18,161 @@ import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powe
 import * as bson from 'bson';
 import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
 import { MongoBucketStorage } from '../../MongoBucketStorage.js';
-import {
-  MongoSyncBucketStorageCheckpoint,
-  MongoSyncBucketStorageContext
-} from '../common/MongoSyncBucketStorageContext.js';
-import { CommonSourceTableDocument, SourceKey } from '../models.js';
-import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
+import { SourceKey } from '../models.js';
 import { MongoChecksums } from '../MongoChecksums.js';
 import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
-import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
 import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
-import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument, loadBucketDataDocumentV1 } from './models.js';
+import {
+  BucketDataDocumentV1,
+  BucketDataKeyV1,
+  BucketStateDocument,
+  loadBucketDataDocumentV1,
+  SyncRuleDocumentV1
+} from './models.js';
 import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js';
 import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
 import { MongoCompactorV1 } from './MongoCompactorV1.js';
 import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 
+export interface MongoSyncBucketStorageContextV1 {
+  db: VersionedPowerSyncMongoV1;
+  replicationStreamId: number;
+}
+
 export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
-  // Declare types to be more specific
   declare readonly db: VersionedPowerSyncMongoV1;
   declare readonly checksums: MongoChecksumsV1;
 
   constructor(
     factory: MongoBucketStorage,
-    group_id: number,
-    sync_rules: MongoPersistedSyncRulesContent,
-    slot_name: string,
+    replicationStreamId: number,
+    replicationStream: MongoPersistedReplicationStream,
+    replicationStreamName: string,
     writeCheckpointMode: storage.WriteCheckpointMode | undefined,
     options: MongoSyncBucketStorageOptions
   ) {
-    super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+    super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
   }
 
   protected async initializeVersionStorage(): Promise<void> {}
 
-  protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch {
-    return new MongoBucketBatchV1(batchOptions);
+  protected async fetchPersistedOpHead(): Promise<InternalOpId | null> {
+    const doc = (await this.db.sync_rules.findOne(
+      { _id: this.replicationStreamId },
+      { projection: { keepalive_op: 1, last_checkpoint: 1 } }
+    )) as SyncRuleDocumentV1;
+    // keepalive_op covers ops not yet in a checkpoint (cleared once checkpointed),
+    // so the head is the max of the two.
+    const keepaliveOp = doc?.keepalive_op == null ? null : BigInt(doc.keepalive_op);
+    const lastCheckpoint = doc?.last_checkpoint ?? null;
+    if (keepaliveOp == null && lastCheckpoint == null) {
+      return null;
+    }
+    return (keepaliveOp ?? 0n) > (lastCheckpoint ?? 0n) ? keepaliveOp : lastCheckpoint;
+  }
+
+  protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
+    const doc = (await this.db.sync_rules.findOne(
+      {
+        _id: this.replicationStreamId
+      },
+      { projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }
+    )) as SyncRuleDocumentV1;
+
+    return new MongoBucketBatchV1({
+      ...this.writerBatchOptions(options),
+      // Resume from the last consistent checkpoint, or the in-progress snapshot position if it is newer.
+      resumeFromLsn: maxLsn(doc?.last_checkpoint_lsn, doc?.snapshot_lsn),
+      keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null
+    });
+  }
+
+  protected async fetchCheckpointState(
+    session: mongo.ClientSession
+  ): Promise<{ checkpoint: bigint; lsn: string | null } | null> {
+    const doc = (await this.db.sync_rules.findOne(
+      { _id: this.replicationStreamId },
+      {
+        session,
+        projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
+      }
+    )) as SyncRuleDocumentV1;
+    if (!doc?.snapshot_done || ![storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED].includes(doc.state)) {
+      return null;
+    }
+    return {
+      checkpoint: doc.last_checkpoint ?? 0n,
+      lsn: doc.last_checkpoint_lsn ?? null
+    };
+  }
+
+  protected async terminateSyncRuleState(): Promise<void> {
+    await this.db.sync_rules.updateOne(
+      {
+        _id: this.replicationStreamId
+      },
+      {
+        $set: {
+          state: storage.SyncRuleState.TERMINATED,
+          persisted_lsn: null,
+          snapshot_done: false
+        }
+      }
+    );
+  }
+
+  protected async getStatusImpl(): Promise<storage.ReplicationStreamStatus> {
+    const doc = (await this.db.sync_rules.findOne(
+      {
+        _id: this.replicationStreamId
+      },
+      {
+        projection: {
+          snapshot_done: 1,
+          last_checkpoint_lsn: 1,
+          snapshot_lsn: 1
+        }
+      }
+    )) as SyncRuleDocumentV1;
+    if (doc == null) {
+      throw new ServiceAssertionError('Cannot find replication stream status');
+    }
+
+    return {
+      snapshotDone: doc.snapshot_done && doc.last_checkpoint_lsn != null,
+      resumeLsn: maxLsn(doc.snapshot_lsn, doc.last_checkpoint_lsn)
+    };
+  }
+
+  protected async clearSyncRuleState(): Promise<void> {
+    await this.db.sync_rules.updateOne(
+      {
+        _id: this.replicationStreamId
+      },
+      {
+        $set: {
+          snapshot_done: false,
+          persisted_lsn: null,
+          last_checkpoint_lsn: null,
+          last_checkpoint: null,
+          no_checkpoint_before: null
+        },
+        $unset: {
+          snapshot_lsn: 1
+        }
+      },
+      { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
+    );
   }
 
   protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums {
-    return new MongoChecksumsV1(this.db, this.group_id, {
+    return new MongoChecksumsV1(this.db, this.replicationStreamId, {
       ...options.checksumOptions,
-      storageConfig: options?.storageConfig,
-      mapping: this.sync_rules.mapping
+      storageConfig: options?.storageConfig
     });
   }
 
@@ -72,26 +184,13 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     checkpoint: InternalOpId,
     options: storage.CompactOptions
   ): MongoParameterCompactor {
-    return new MongoParameterCompactorV1(this.db, this.group_id, checkpoint, options);
-  }
-
-  protected sourceTableBaseId(): Partial<CommonSourceTableDocument> {
-    return { group_id: this.group_id };
+    return new MongoParameterCompactorV1(this.db, this.replicationStreamId, checkpoint, options);
   }
 
-  protected augmentCreatedSourceTableDocument(
-    _createDoc: CommonSourceTableDocument,
-    _options: storage.ResolveTableOptions,
-    _candidateSourceTable: storage.SourceTable
-  ): void {}
-
-  protected async initializeResolvedSourceRecords(_sourceTableId: bson.ObjectId): Promise<void> {}
-
-  protected override get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1> {
+  protected get versionContext(): MongoSyncBucketStorageContextV1 {
     return {
       db: this.db,
-      group_id: this.group_id,
-      mapping: this.mapping
+      replicationStreamId: this.replicationStreamId
     };
   }
 
@@ -117,7 +216,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.bucket_data.deleteMany(
           {
-            _id: idPrefixFilter<BucketDataKeyV1>({ g: this.group_id }, ['b', 'o'])
+            _id: idPrefixFilter<BucketDataKeyV1>({ g: this.replicationStreamId }, ['b', 'o'])
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -131,7 +230,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.parameterIndexV1.deleteMany(
           {
-            'key.g': this.group_id
+            'key.g': this.replicationStreamId
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -145,7 +244,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.sourceRecordsV1.deleteMany(
           {
-            _id: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
+            _id: idPrefixFilter<SourceKey>({ g: this.replicationStreamId }, ['t', 'k'])
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -159,7 +258,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.bucketStateV1.deleteMany(
           {
-            _id: idPrefixFilter<BucketStateDocument['_id']>({ g: this.group_id }, ['b'])
+            _id: idPrefixFilter<BucketStateDocument['_id']>({ g: this.replicationStreamId }, ['b'])
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -171,9 +270,9 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     await this.clearDeleteMany(
       'source tables',
       () =>
-        this.db.commonSourceTables(this.group_id).deleteMany(
+        this.db.sourceTablesV1(this.replicationStreamId).deleteMany(
           {
-            group_id: this.group_id
+            group_id: this.replicationStreamId
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -195,7 +294,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
 }
 
 export async function getParameterSetsV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   checkpoint: MongoSyncBucketStorageCheckpoint,
   lookups: ScopedParameterLookup[],
   limit: number
@@ -211,7 +310,7 @@ export async function getParameterSetsV1(
         [
           {
             $match: {
-              'key.g': ctx.group_id,
+              'key.g': ctx.replicationStreamId,
               lookup: { $in: lookupFilter },
               _id: { $lte: checkpoint.checkpoint }
             }
@@ -269,7 +368,7 @@ export async function getParameterSetsV1(
 }
 
 export async function* getBucketDataBatchV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   checkpoint: utils.InternalOpId,
   dataBuckets: storage.BucketDataRequest[],
   options?: storage.BucketDataBatchOptions
@@ -288,12 +387,12 @@ export async function* getBucketDataBatchV1(
     filters.push({
       _id: {
         $gt: {
-          g: ctx.group_id,
+          g: ctx.replicationStreamId,
           b: name,
           o: start
         },
         $lte: {
-          g: ctx.group_id,
+          g: ctx.replicationStreamId,
           b: name,
           o: end as any
         }
@@ -304,6 +403,8 @@ export async function* getBucketDataBatchV1(
   const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
   const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
 
+  // raw: true returns Buffers, but the driver typing doesn't reflect that
+  // without an explicit cast to FindCursor<Buffer>.
   const cursor = ctx.db.bucket_data.find(
     {
       $or: filters
@@ -386,14 +487,14 @@ export async function* getBucketDataBatchV1(
 }
 
 export async function getDataBucketChangesV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   options: GetCheckpointChangesOptions
 ): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
   const limit = 1000;
   const bucketStateUpdates = await ctx.db.bucketStateV1
     .find(
       {
-        '_id.g': ctx.group_id,
+        '_id.g': ctx.replicationStreamId,
         last_op: { $gt: options.lastCheckpoint.checkpoint }
       },
       {
@@ -417,7 +518,7 @@ export async function getDataBucketChangesV1(
 }
 
 export async function getParameterBucketChangesV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   options: GetCheckpointChangesOptions
 ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
   const limit = 1000;
@@ -425,7 +526,7 @@ export async function getParameterBucketChangesV1(
     .find(
       {
         _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
-        'key.g': ctx.group_id
+        'key.g': ctx.replicationStreamId
       },
       {
         projection: {

package/src/storage/implementation/v1/PersistedBatchV1.ts

@@ -3,9 +3,8 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import * as bson from 'bson';
 
-import { BucketDataSource } from '@powersync/service-sync-rules';
+import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
 import { mongoTableId } from '../../../utils/util.js';
-import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
 import { EMPTY_DATA } from '../MongoBucketBatchShared.js';
 import {
   BucketStateUpdate,

package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts

@@ -1,7 +1,13 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
 import { CommonSourceTableDocument } from '../models.js';
-import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js';
+import {
+  BucketDataDocumentV1,
+  BucketParameterDocument,
+  BucketStateDocumentV1,
+  CurrentDataDocument,
+  SourceTableDocumentV1
+} from './models.js';
 
 export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
   get sourceRecordsV1(): mongo.Collection<CurrentDataDocument> {
@@ -16,6 +22,10 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
     return this.upstream.source_tables as any as mongo.Collection<CommonSourceTableDocument>;
   }
 
+  sourceTablesV1(_replicationStreamId: number): mongo.Collection<SourceTableDocumentV1> {
+    return this.upstream.source_tables as any as mongo.Collection<SourceTableDocumentV1>;
+  }
+
   async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
 
   get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {

package/src/storage/implementation/v1/models.ts

@@ -1,3 +1,4 @@
+import { SerializedSyncPlan } from '@powersync/service-core';
 import * as bson from 'bson';
 import { BucketDataDoc } from '../common/BucketDataDoc.js';
 import {
@@ -8,6 +9,8 @@ import {
   LEGACY_BUCKET_DATA_DEFINITION_ID,
   SourceKey,
   SourceTableDocument,
+  SyncRuleCheckpointFields,
+  SyncRuleDocumentBase,
   TaggedBucketParameterDocument
 } from '../models.js';
 
@@ -75,6 +78,28 @@ export interface SourceTableDocumentV1 extends SourceTableDocument {
   group_id: number;
 }
 
+export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields {
+  content: string;
+  serialized_plan?: SerializedSyncPlan | null;
+
+  /**
+   * True if initial snapshot has been replicated.
+   *
+   * Can only be false if state == PROCESSING.
+   */
+  snapshot_done: boolean;
+
+  /**
+   * Goes together with no_checkpoint_before.
+   *
+   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
+   * then the checkpoint must be equal to this keepalive_op.
+   *
+   * string for legacy reasons - it's always a number.
+   */
+  keepalive_op: string | null;
+}
+
 export interface BucketStateDocumentV1 extends BucketStateDocumentBase {
   _id: BucketStateDocumentBase['_id'] & {
     g: number;