@powersync/service-module-mongodb-storage 0.17.0 → 0.18.1

package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts

@@ -18,16 +18,12 @@ import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powe
 import * as bson from 'bson';
 import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
 import { MongoBucketStorage } from '../../MongoBucketStorage.js';
-import {
-  MongoSyncBucketStorageCheckpoint,
-  MongoSyncBucketStorageContext
-} from '../common/MongoSyncBucketStorageContext.js';
+import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
 import { SourceKey } from '../models.js';
-import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
 import { MongoChecksums } from '../MongoChecksums.js';
 import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
-import { MongoPersistedSyncRulesContentV1 } from '../MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
 import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
 import {
   BucketDataDocumentV1,
@@ -42,33 +38,64 @@ import { MongoCompactorV1 } from './MongoCompactorV1.js';
 import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 
+export interface MongoSyncBucketStorageContextV1 {
+  db: VersionedPowerSyncMongoV1;
+  replicationStreamId: number;
+}
+
 export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
-  // Declare types to be more specific
   declare readonly db: VersionedPowerSyncMongoV1;
   declare readonly checksums: MongoChecksumsV1;
 
   constructor(
     factory: MongoBucketStorage,
-    group_id: number,
-    sync_rules: MongoPersistedSyncRulesContentV1,
-    slot_name: string,
+    replicationStreamId: number,
+    replicationStream: MongoPersistedReplicationStream,
+    replicationStreamName: string,
     writeCheckpointMode: storage.WriteCheckpointMode | undefined,
     options: MongoSyncBucketStorageOptions
   ) {
-    super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+    super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
   }
 
   protected async initializeVersionStorage(): Promise<void> {}
 
-  protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch {
-    return new MongoBucketBatchV1(batchOptions);
+  protected async fetchPersistedOpHead(): Promise<InternalOpId | null> {
+    const doc = (await this.db.sync_rules.findOne(
+      { _id: this.replicationStreamId },
+      { projection: { keepalive_op: 1, last_checkpoint: 1 } }
+    )) as SyncRuleDocumentV1;
+    // keepalive_op covers ops not yet in a checkpoint (cleared once checkpointed),
+    // so the head is the max of the two.
+    const keepaliveOp = doc?.keepalive_op == null ? null : BigInt(doc.keepalive_op);
+    const lastCheckpoint = doc?.last_checkpoint ?? null;
+    if (keepaliveOp == null && lastCheckpoint == null) {
+      return null;
+    }
+    return (keepaliveOp ?? 0n) > (lastCheckpoint ?? 0n) ? keepaliveOp : lastCheckpoint;
+  }
+
+  protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
+    const doc = (await this.db.sync_rules.findOne(
+      {
+        _id: this.replicationStreamId
+      },
+      { projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }
+    )) as SyncRuleDocumentV1;
+
+    return new MongoBucketBatchV1({
+      ...this.writerBatchOptions(options),
+      // Resume from the last consistent checkpoint, or the in-progress snapshot position if it is newer.
+      resumeFromLsn: maxLsn(doc?.last_checkpoint_lsn, doc?.snapshot_lsn),
+      keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null
+    });
   }
 
   protected async fetchCheckpointState(
     session: mongo.ClientSession
   ): Promise<{ checkpoint: bigint; lsn: string | null } | null> {
     const doc = (await this.db.sync_rules.findOne(
-      { _id: this.group_id },
+      { _id: this.replicationStreamId },
       {
         session,
         projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
@@ -83,26 +110,10 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     };
   }
 
-  protected async getWriterSyncState() {
-    const doc = (await this.db.sync_rules.findOne(
-      {
-        _id: this.group_id
-      },
-      { projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }
-    )) as SyncRuleDocumentV1;
-    const checkpointLsn = doc?.last_checkpoint_lsn ?? null;
-    return {
-      lastCheckpointLsn: checkpointLsn,
-      resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
-      keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
-      syncConfigId: null
-    };
-  }
-
   protected async terminateSyncRuleState(): Promise<void> {
     await this.db.sync_rules.updateOne(
       {
-        _id: this.group_id
+        _id: this.replicationStreamId
       },
       {
         $set: {
@@ -114,18 +125,16 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     );
   }
 
-  protected async getStatusImpl(): Promise<storage.SyncRuleStatus> {
+  protected async getStatusImpl(): Promise<storage.ReplicationStreamStatus> {
     const doc = (await this.db.sync_rules.findOne(
       {
-        _id: this.group_id
+        _id: this.replicationStreamId
       },
       {
         projection: {
           snapshot_done: 1,
           last_checkpoint_lsn: 1,
-          state: 1,
-          snapshot_lsn: 1,
-          keepalive_op: 1
+          snapshot_lsn: 1
         }
       }
     )) as SyncRuleDocumentV1;
@@ -134,18 +143,15 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     }
 
     return {
-      snapshot_done: doc.snapshot_done,
-      snapshot_lsn: doc.snapshot_lsn ?? null,
-      active: doc.state == storage.SyncRuleState.ACTIVE,
-      checkpoint_lsn: doc.last_checkpoint_lsn,
-      keepalive_op: doc.keepalive_op == null ? null : BigInt(doc.keepalive_op)
+      snapshotDone: doc.snapshot_done && doc.last_checkpoint_lsn != null,
+      resumeLsn: maxLsn(doc.snapshot_lsn, doc.last_checkpoint_lsn)
     };
   }
 
   protected async clearSyncRuleState(): Promise<void> {
     await this.db.sync_rules.updateOne(
       {
-        _id: this.group_id
+        _id: this.replicationStreamId
       },
       {
         $set: {
@@ -164,10 +170,9 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
   }
 
   protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums {
-    return new MongoChecksumsV1(this.db, this.group_id, {
+    return new MongoChecksumsV1(this.db, this.replicationStreamId, {
       ...options.checksumOptions,
-      storageConfig: options?.storageConfig,
-      mapping: this.sync_rules.mapping
+      storageConfig: options?.storageConfig
     });
   }
 
@@ -179,14 +184,13 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     checkpoint: InternalOpId,
     options: storage.CompactOptions
   ): MongoParameterCompactor {
-    return new MongoParameterCompactorV1(this.db, this.group_id, checkpoint, options);
+    return new MongoParameterCompactorV1(this.db, this.replicationStreamId, checkpoint, options);
   }
 
-  protected override get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1> {
+  protected get versionContext(): MongoSyncBucketStorageContextV1 {
     return {
       db: this.db,
-      group_id: this.group_id,
-      mapping: this.mapping
+      replicationStreamId: this.replicationStreamId
     };
   }
 
@@ -212,7 +216,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.bucket_data.deleteMany(
           {
-            _id: idPrefixFilter<BucketDataKeyV1>({ g: this.group_id }, ['b', 'o'])
+            _id: idPrefixFilter<BucketDataKeyV1>({ g: this.replicationStreamId }, ['b', 'o'])
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -226,7 +230,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.parameterIndexV1.deleteMany(
           {
-            'key.g': this.group_id
+            'key.g': this.replicationStreamId
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -240,7 +244,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.sourceRecordsV1.deleteMany(
           {
-            _id: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
+            _id: idPrefixFilter<SourceKey>({ g: this.replicationStreamId }, ['t', 'k'])
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -254,7 +258,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
       () =>
         this.db.bucketStateV1.deleteMany(
           {
-            _id: idPrefixFilter<BucketStateDocument['_id']>({ g: this.group_id }, ['b'])
+            _id: idPrefixFilter<BucketStateDocument['_id']>({ g: this.replicationStreamId }, ['b'])
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -266,9 +270,9 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     await this.clearDeleteMany(
       'source tables',
       () =>
-        this.db.commonSourceTables(this.group_id).deleteMany(
+        this.db.sourceTablesV1(this.replicationStreamId).deleteMany(
           {
-            group_id: this.group_id
+            group_id: this.replicationStreamId
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
@@ -290,7 +294,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
 }
 
 export async function getParameterSetsV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   checkpoint: MongoSyncBucketStorageCheckpoint,
   lookups: ScopedParameterLookup[],
   limit: number
@@ -306,7 +310,7 @@ export async function getParameterSetsV1(
         [
           {
             $match: {
-              'key.g': ctx.group_id,
+              'key.g': ctx.replicationStreamId,
               lookup: { $in: lookupFilter },
               _id: { $lte: checkpoint.checkpoint }
             }
@@ -364,7 +368,7 @@ export async function getParameterSetsV1(
 }
 
 export async function* getBucketDataBatchV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   checkpoint: utils.InternalOpId,
   dataBuckets: storage.BucketDataRequest[],
   options?: storage.BucketDataBatchOptions
@@ -383,12 +387,12 @@ export async function* getBucketDataBatchV1(
     filters.push({
       _id: {
         $gt: {
-          g: ctx.group_id,
+          g: ctx.replicationStreamId,
           b: name,
           o: start
         },
         $lte: {
-          g: ctx.group_id,
+          g: ctx.replicationStreamId,
           b: name,
           o: end as any
         }
@@ -399,6 +403,8 @@ export async function* getBucketDataBatchV1(
   const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
   const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
 
+  // raw: true returns Buffers, but the driver typing doesn't reflect that
+  // without an explicit cast to FindCursor<Buffer>.
   const cursor = ctx.db.bucket_data.find(
     {
       $or: filters
@@ -481,14 +487,14 @@ export async function* getBucketDataBatchV1(
 }
 
 export async function getDataBucketChangesV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   options: GetCheckpointChangesOptions
 ): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
   const limit = 1000;
   const bucketStateUpdates = await ctx.db.bucketStateV1
     .find(
       {
-        '_id.g': ctx.group_id,
+        '_id.g': ctx.replicationStreamId,
         last_op: { $gt: options.lastCheckpoint.checkpoint }
       },
       {
@@ -512,7 +518,7 @@ export async function getDataBucketChangesV1(
 }
 
 export async function getParameterBucketChangesV1(
-  ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>,
+  ctx: MongoSyncBucketStorageContextV1,
   options: GetCheckpointChangesOptions
 ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
   const limit = 1000;
@@ -520,7 +526,7 @@ export async function getParameterBucketChangesV1(
     .find(
       {
         _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
-        'key.g': ctx.group_id
+        'key.g': ctx.replicationStreamId
       },
       {
         projection: {

package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts

@@ -1,7 +1,13 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
 import { CommonSourceTableDocument } from '../models.js';
-import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js';
+import {
+  BucketDataDocumentV1,
+  BucketParameterDocument,
+  BucketStateDocumentV1,
+  CurrentDataDocument,
+  SourceTableDocumentV1
+} from './models.js';
 
 export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
   get sourceRecordsV1(): mongo.Collection<CurrentDataDocument> {
@@ -16,6 +22,10 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
     return this.upstream.source_tables as any as mongo.Collection<CommonSourceTableDocument>;
   }
 
+  sourceTablesV1(_replicationStreamId: number): mongo.Collection<SourceTableDocumentV1> {
+    return this.upstream.source_tables as any as mongo.Collection<SourceTableDocumentV1>;
+  }
+
   async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
 
   get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {

package/src/storage/implementation/v1/models.ts

@@ -78,7 +78,7 @@ export interface SourceTableDocumentV1 extends SourceTableDocument {
   group_id: number;
 }
 
-export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields<string | null> {
+export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields {
   content: string;
   serialized_plan?: SerializedSyncPlan | null;
 
@@ -88,6 +88,16 @@ export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckp
    * Can only be false if state == PROCESSING.
    */
   snapshot_done: boolean;
+
+  /**
+   * Goes together with no_checkpoint_before.
+   *
+   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
+   * then the checkpoint must be equal to this keepalive_op.
+   *
+   * string for legacy reasons - it's always a number.
+   */
+  keepalive_op: string | null;
 }
 
 export interface BucketStateDocumentV1 extends BucketStateDocumentBase {