@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -1,5 +1,6 @@
 import crypto from 'crypto';
 
+import { mongo } from '@powersync/lib-service-mongodb';
 import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { VersionedPowerSyncMongo } from './db.js';
@@ -11,13 +12,17 @@ import { VersionedPowerSyncMongo } from './db.js';
 export class MongoSyncRulesLock implements storage.ReplicationLock {
   private readonly refreshInterval: NodeJS.Timeout;
 
+  /**
+   * @param session optional session to create the lock within another transaction
+   */
   static async createLock(
     db: VersionedPowerSyncMongo,
-    sync_rules: storage.PersistedSyncRulesContent
+    sync_rules: storage.PersistedReplicationStream,
+    session?: mongo.ClientSession
   ): Promise<MongoSyncRulesLock> {
     const lockId = crypto.randomBytes(8).toString('hex');
     const doc = await db.sync_rules.findOneAndUpdate(
-      { _id: sync_rules.id, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
+      { _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
       {
         $set: {
           lock: {
@@ -28,13 +33,17 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
       },
       {
         projection: { lock: 1 },
-        returnDocument: 'before'
+        returnDocument: 'before',
+        session
       }
     );
 
     if (doc == null) {
       // Query the existing lock to get the expiration time (best effort - it may have been released in the meantime).
-      const heldLock = await db.sync_rules.findOne({ _id: sync_rules.id }, { projection: { lock: 1 } });
+      const heldLock = await db.sync_rules.findOne(
+        { _id: sync_rules.replicationStreamId },
+        { projection: { lock: 1 }, session }
+      );
       if (heldLock?.lock?.expires_at) {
         throw new ServiceError(
           ErrorCode.PSYNC_S1003,
@@ -45,7 +54,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
       }
     }
     sync_rules.logger.info(`Locked replication stream for processing`);
-    return new MongoSyncRulesLock(db, sync_rules.id, lockId, sync_rules.logger);
+    return new MongoSyncRulesLock(db, sync_rules.replicationStreamId, lockId, sync_rules.logger);
   }
 
   constructor(

package/src/storage/implementation/SyncRuleStateUpdate.ts

@@ -0,0 +1,38 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+
+/**
+ * Update pipeline to update replication stream status, covering all storage versions.
+ *
+ * Roughly equivalent to:
+ *   $set: {
+ *     state: state,
+ *     'sync_configs.$[].state': state
+ *   }
+ *
+ * The difference is that this also handles v1 storage cases, where `sync_configs` is not present.
+ */
+export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo.Document[] {
+  return [
+    {
+      $set: {
+        state,
+        sync_configs: {
+          $cond: [
+            { $isArray: '$sync_configs' },
+            {
+              $map: {
+                input: '$sync_configs',
+                as: 'config',
+                in: {
+                  $mergeObjects: ['$$config', { state }]
+                }
+              }
+            },
+            '$$REMOVE'
+          ]
+        }
+      }
+    }
+  ];
+}

package/src/storage/implementation/common/BucketDataDoc.ts

@@ -1,5 +1,5 @@
 import { InternalOpId } from '@powersync/service-core';
-import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
+import { BucketDefinitionId } from '@powersync/service-sync-rules';
 import { BucketDataProperties } from '../models.js';
 
 /**

package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts

@@ -0,0 +1,7 @@
+import { InternalOpId } from '@powersync/service-core';
+import * as bson from 'bson';
+
+export interface MongoSyncBucketStorageCheckpoint {
+  checkpoint: InternalOpId;
+  snapshotTime: bson.Timestamp;
+}

package/src/storage/implementation/common/PersistedBatch.ts

@@ -1,12 +1,11 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { BucketDataSource, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import { BucketDataSource, BucketDefinitionId, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 
 import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework';
-import { InternalOpId, storage, utils } from '@powersync/service-core';
+import { BucketDefinitionMapping, InternalOpId, storage, utils } from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
-import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
 import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
 import { MongoIdSequence } from '../MongoIdSequence.js';
 import type { VersionedPowerSyncMongo } from '../db.js';
@@ -97,7 +96,7 @@ export abstract class PersistedBatch {
   }
 
   saveBucketData(options: SaveBucketDataOptions) {
-    const remaining_buckets = new Map<string, SaveBucketDataOptions['before_buckets'][number]>();
+    const remaining_buckets = new Map<string, SourceRecordBucketState>();
     for (let bucket of options.before_buckets) {
       const mapped: SourceRecordBucketState = {
         bucket: bucket.bucket,
@@ -105,6 +104,11 @@ export abstract class PersistedBatch {
         id: bucket.id,
         table: bucket.table
       };
+      if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId!)) {
+        // The bucket definition is not active anymore.
+        // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
+        continue;
+      }
       remaining_buckets.set(currentBucketKey(mapped), mapped);
     }
 

package/src/storage/implementation/common/SourceRecordStore.ts

@@ -1,9 +1,8 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
-import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import { BucketDefinitionId, EvaluatedParameters, EvaluatedRow, ParameterIndexId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
 
 export interface SourceRecordLookupEntry {
   sourceTableId: bson.ObjectId;

package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts

@@ -67,7 +67,7 @@ export abstract class BaseVersionedPowerSyncMongo {
   }
 
   protected async listCollectionsByPrefix<T extends mongo.Document>(prefix: string): Promise<mongo.Collection<T>[]> {
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+    const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
 
     return collections
       .filter((collection) => collection.name.startsWith(prefix))

package/src/storage/implementation/createMongoSyncBucketStorage.ts

@@ -1,6 +1,6 @@
 import { storage } from '@powersync/service-core';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
 import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
 import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
 import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
@@ -11,15 +11,29 @@ export type { MongoSyncBucketStorage };
 
 export function createMongoSyncBucketStorage(
   factory: MongoBucketStorage,
-  group_id: number,
-  sync_rules: MongoPersistedSyncRulesContent,
-  slot_name: string,
+  replicationStreamId: number,
+  replicationStream: MongoPersistedReplicationStream,
+  replicationStreamName: string,
   writeCheckpointMode: storage.WriteCheckpointMode | undefined,
   options: MongoSyncBucketStorageOptions
 ): MongoSyncBucketStorage {
-  if (sync_rules.getStorageConfig().incrementalReprocessing) {
-    return new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  if (replicationStream.getStorageConfig().incrementalReprocessing) {
+    return new MongoSyncBucketStorageV3(
+      factory,
+      replicationStreamId,
+      replicationStream,
+      replicationStreamName,
+      writeCheckpointMode,
+      options
+    );
   }
 
-  return new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  return new MongoSyncBucketStorageV1(
+    factory,
+    replicationStreamId,
+    replicationStream,
+    replicationStreamName,
+    writeCheckpointMode,
+    options
+  );
 }

package/src/storage/implementation/db.ts

@@ -14,7 +14,7 @@ import {
   InstanceDocument,
   SourceTableDocument,
   StorageConfig,
-  SyncRuleDocument,
+  SyncRuleDocumentBase,
   WriteCheckpointDocument
 } from './models.js';
 import {
@@ -41,7 +41,7 @@ export class PowerSyncMongo {
   readonly bucket_data: mongo.Collection<BucketDataDocumentV1>;
   readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
   readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
-  readonly sync_rules: mongo.Collection<SyncRuleDocument>;
+  readonly sync_rules: mongo.Collection<SyncRuleDocumentBase>;
   readonly source_tables: mongo.Collection<SourceTableDocument>;
   readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
   readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
@@ -77,8 +77,11 @@ export class PowerSyncMongo {
     this.connection_report_events = this.db.collection('connection_report_events');
   }
 
+  versioned(storageConfig: StorageConfig & { incrementalReprocessing: true }): VersionedPowerSyncMongoV3;
+  versioned(storageConfig: StorageConfig & { incrementalReprocessing: false }): VersionedPowerSyncMongoV1;
+  versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo;
   versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo {
-    if (storageConfig.incrementalReprocessing) {
+    if (storageConfig.version >= storage.STORAGE_VERSION_3) {
       return new VersionedPowerSyncMongoV3(this, storageConfig);
     }
 
@@ -90,22 +93,18 @@ export class PowerSyncMongo {
    */
   async listBucketDataCollectionsV3(groupId?: number): Promise<mongo.Collection<BucketDataDocumentV3>[]> {
     const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
-
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<BucketDataDocumentV3>(collection.name));
+    return this.collectionsByPrefix<BucketDataDocumentV3>(prefix);
   }
 
   /**
    * Not safe for user-provided prefix - only for hardcoded values.
    */
-  private async collectionsByPrefix(prefix: string): Promise<mongo.Collection<never>[]> {
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+  private async collectionsByPrefix<T extends mongo.Document = never>(prefix: string): Promise<mongo.Collection<T>[]> {
+    const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
 
     return collections
       .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<never>(collection.name));
+      .map((collection) => this.db.collection<T>(collection.name));
   }
 
   /**
@@ -141,16 +140,12 @@ export class PowerSyncMongo {
   async listSourceTableCollections(
     replicationStreamId?: number
   ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
-    const filter =
-      replicationStreamId == null
-        ? { name: new RegExp('^source_table_') }
-        : { name: this.sourceTableCollectionName(replicationStreamId) };
-    const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId);
-    const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray();
-
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<CommonSourceTableDocument>(collection.name));
+    if (replicationStreamId == null) {
+      return this.collectionsByPrefix('source_table_');
+    }
+    const name = this.sourceTableCollectionName(replicationStreamId);
+    const collections = await this.db.listCollections({ name }, { nameOnly: true }).toArray();
+    return collections.map((c) => this.db.collection<CommonSourceTableDocument>(c.name));
   }
 
   /**

package/src/storage/implementation/models.ts

@@ -1,11 +1,10 @@
 import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
-import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core';
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import { InternalOpId, storage } from '@powersync/service-core';
+import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules';
 import { event_types } from '@powersync/service-types';
 import * as bson from 'bson';
-import { ParameterIndexId } from './BucketDefinitionMapping.js';
 import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
-import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js';
+import type { CurrentDataDocumentV3, RecordedLookupV3 } from './v3/models.js';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -157,18 +156,14 @@ export interface IdSequenceDocument {
   op_id: bigint;
 }
 
-export interface SyncRuleDocument {
+/**
+ * Base for sync_rules collection.
+ */
+export interface SyncRuleDocumentBase {
   _id: number;
 
   state: storage.SyncRuleState;
 
-  /**
-   * True if initial snapshot has been replicated.
-   *
-   * Can only be false if state == PROCESSING.
-   */
-  snapshot_done: boolean;
-
   /**
    * This is now used for "resumeLsn".
    *
@@ -182,31 +177,6 @@ export interface SyncRuleDocument {
    */
   snapshot_lsn: string | undefined;
 
-  /**
-   * The last consistent checkpoint.
-   *
-   * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
-   */
-  last_checkpoint: bigint | null;
-
-  /**
-   * The LSN associated with the last consistent checkpoint.
-   */
-  last_checkpoint_lsn: string | null;
-
-  /**
-   * If set, no new checkpoints may be created < this value.
-   */
-  no_checkpoint_before: string | null;
-
-  /**
-   * Goes together with no_checkpoint_before.
-   *
-   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
-   * then the checkpoint must be equal to this keepalive_op.
-   */
-  keepalive_op: string | null;
-
   slot_name: string | null;
 
   /**
@@ -230,23 +200,6 @@ export interface SyncRuleDocument {
 
   last_fatal_error_ts: Date | null;
 
-  content: string;
-  serialized_plan?: SerializedSyncPlan | null;
-
-  /**
-   * Required for V3+ storage.
-   */
-  rule_mapping?: {
-    /**
-     * Map of uniqueName -> id, unique per replication stream.
-     */
-    definitions: Record<string, string>;
-    /**
-     * Map of (lookupName, queryId) -> id, unique per replication stream.
-     */
-    parameter_indexes: Record<string, string>;
-  };
-
   lock?: {
     id: string;
     expires_at: Date;
@@ -255,6 +208,25 @@ export interface SyncRuleDocument {
   storage_version?: number;
 }
 
+export interface SyncRuleCheckpointFields {
+  /**
+   * The last consistent checkpoint.
+   *
+   * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
+   */
+  last_checkpoint: bigint | null;
+
+  /**
+   * The LSN associated with the last consistent checkpoint.
+   */
+  last_checkpoint_lsn: string | null;
+
+  /**
+   * If set, no new checkpoints may be created < this value.
+   */
+  no_checkpoint_before: string | null;
+}
+
 export interface StorageConfig extends storage.StorageVersionConfig {
   /**
    * When true, bucket_data.checksum is guaranteed to be persisted as a Long.
@@ -289,11 +261,6 @@ export interface CheckpointEventDocument {
   _id: bson.ObjectId;
 }
 
-export type SyncRuleCheckpointState = Pick<
-  SyncRuleDocument,
-  'last_checkpoint' | 'last_checkpoint_lsn' | '_id' | 'state'
->;
-
 export interface CustomWriteCheckpointDocument {
   _id: bson.ObjectId;
   user_id: string;
@@ -329,6 +296,6 @@ export interface InstanceDocument {
 export interface ClientConnectionDocument extends event_types.ClientConnection {}
 
 export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id'];
-export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3;
+export type CommonCurrentBucket = CurrentBucket;
 export type CommonCurrentLookup = bson.Binary | RecordedLookupV3;
-export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3;
+export type CommonSourceTableDocument = Pick<SourceTableDocumentV1, '_id' | 'snapshot_status'>;