@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

package/src/storage/MongoBucketStorage.ts

@@ -1,4 +1,4 @@
-import { GetIntanceOptions, storage } from '@powersync/service-core';
+import { GetIntanceOptions, LEGACY_STORAGE_VERSION, storage } from '@powersync/service-core';
 
 import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
@@ -6,15 +6,25 @@ import { v4 as uuid } from 'uuid';
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 
+import { ObjectId } from 'bson';
 import { generateSlotName } from '../utils/util.js';
+import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js';
+import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
+import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { PowerSyncMongo } from './implementation/db.js';
-import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js';
+import { getMongoStorageConfig, StorageConfig, SyncRuleDocumentBase } from './implementation/models.js';
 import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
-import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
-import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
+import {
+  MongoPersistedSyncRulesContentV1,
+  MongoPersistedSyncRulesContentV3
+} from './implementation/MongoPersistedSyncRulesContent.js';
+import { syncRuleStateUpdatePipeline } from './implementation/SyncRuleStateUpdate.js';
+import { SyncRuleDocumentV1 } from './implementation/v1/models.js';
+import { VersionedPowerSyncMongoV3 } from './implementation/v3/VersionedPowerSyncMongoV3.js';
+import { ReplicationStreamDocumentV3, SyncConfigDefinition } from './storage-index.js';
 
 export interface MongoBucketStorageOptions {
-  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
 }
 
 export class MongoBucketStorage extends storage.BucketStorageFactory {
@@ -52,11 +62,11 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     if ((typeof id as any) == 'bigint') {
       id = Number(id);
     }
-    const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig();
-    const storage = new MongoSyncBucketStorage(
+    const storageConfig = (syncRules as MongoPersistedSyncRulesContentV1).getStorageConfig();
+    const storage = createMongoSyncBucketStorage(
       this,
       id,
-      syncRules as MongoPersistedSyncRulesContent,
+      syncRules as MongoPersistedSyncRulesContentV1,
       slot_name,
       undefined,
       {
@@ -100,7 +110,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     const active = await this.getActiveSyncRulesContent();
 
     if (next != null && next.id == sync_rules_group_id) {
-      // We need to redo the "next" sync rules
+      // We need to redo the "next" replication stream
       await this.updateSyncRules(next.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sync_rules.updateOne(
@@ -108,15 +118,11 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
           _id: next.id,
           state: storage.SyncRuleState.PROCESSING
         },
-        {
-          $set: {
-            state: storage.SyncRuleState.STOP
-          }
-        }
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP)
       );
       await this.db.notifyCheckpoint();
     } else if (next == null && active?.id == sync_rules_group_id) {
-      // Slot removed for "active" sync rules, while there is no "next" one.
+      // Slot removed for "active" replication stream, while there is no "next" one.
       await this.updateSyncRules(active.asUpdateOptions());
 
       // In this case we keep the old one as active for clients, so that that existing clients
@@ -128,45 +134,133 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
           _id: active.id,
           state: storage.SyncRuleState.ACTIVE
         },
-        {
-          $set: {
-            state: storage.SyncRuleState.ERRORED
-          }
-        }
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED)
       );
       await this.db.notifyCheckpoint();
     } else if (next != null && active?.id == sync_rules_group_id) {
-      // Already have next sync rules, but need to stop replicating the active one.
+      // Already have next replication stream, but need to stop replicating the active one.
 
       await this.db.sync_rules.updateOne(
         {
           _id: active.id,
           state: storage.SyncRuleState.ACTIVE
         },
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED)
+      );
+      await this.db.notifyCheckpoint();
+    }
+  }
+
+  private async updateSyncRulesV3(
+    options: storage.UpdateSyncRulesOptions,
+    storageVersion: number,
+    storageConfig: StorageConfig
+  ): Promise<MongoPersistedSyncRulesContentV3> {
+    let rules: MongoPersistedSyncRulesContentV3 | undefined = undefined;
+    const versioned = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3;
+
+    const session = this.session;
+
+    await session.withTransaction(async () => {
+      // Only have a single replication stream with PROCESSING.
+      await this.db.sync_rules.updateMany(
+        {
+          state: storage.SyncRuleState.PROCESSING
+        },
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP),
+        { session }
+      );
+
+      const id_doc = await this.db.op_id_sequence.findOneAndUpdate(
+        {
+          _id: 'sync_rules'
+        },
         {
-          $set: {
-            state: storage.SyncRuleState.ERRORED
+          $inc: {
+            op_id: 1n
           }
+        },
+        {
+          upsert: true,
+          returnDocument: 'after',
+          session
         }
       );
+
+      const id = Number(id_doc!.op_id);
+      const slot_name = generateSlotName(this.slot_name_prefix, id);
+
+      const mapping = BucketDefinitionMapping.fromParsedSyncRules(options.config.parsed);
+
+      const syncConfigDoc: SyncConfigDefinition = {
+        _id: new ObjectId(),
+        replication_stream_id: id,
+        created_at: new Date(),
+        storage_version: storageVersion,
+        content: options.config.yaml,
+        serialized_plan: options.config.plan,
+        rule_mapping: mapping.serialize()
+      };
+      await versioned.syncConfigDefinitions.insertOne(syncConfigDoc, { session });
+
+      const doc: ReplicationStreamDocumentV3 = {
+        _id: id,
+        storage_version: storageVersion,
+        sync_configs: [
+          {
+            _id: syncConfigDoc._id,
+            state: storage.SyncRuleState.PROCESSING,
+            keepalive_op: null,
+            last_checkpoint: null,
+            last_checkpoint_lsn: null,
+            no_checkpoint_before: null,
+            snapshot_done: false
+          }
+        ],
+        snapshot_lsn: undefined,
+        state: storage.SyncRuleState.PROCESSING,
+        slot_name: slot_name,
+        last_checkpoint_ts: null,
+        last_fatal_error: null,
+        last_fatal_error_ts: null,
+        last_keepalive_ts: null
+      };
+
+      await this.db.sync_rules.insertOne(doc, { session });
       await this.db.notifyCheckpoint();
-    }
+      rules = new MongoPersistedSyncRulesContentV3(this.db, doc, syncConfigDoc);
+      if (options.lock) {
+        // The lock is persisted on rules.current_lock
+        await rules.lock(session);
+      }
+    });
+
+    return rules!;
   }
 
-  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
-    const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+  async updateSyncRules(
+    options: storage.UpdateSyncRulesOptions
+  ): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3> {
+    const storageVersion =
+      options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+
     const storageConfig = getMongoStorageConfig(storageVersion);
-    await this.db.initializeStorageVersion(storageConfig);
+    if (storageConfig.incrementalReprocessing) {
+      return this.updateSyncRulesV3(options, storageVersion, storageConfig);
+    }
+
+    let rules: MongoPersistedSyncRulesContentV1 | undefined = undefined;
 
-    let rules: MongoPersistedSyncRulesContent | undefined = undefined;
+    const session = this.session;
 
-    await this.session.withTransaction(async () => {
-      // Only have a single set of sync rules with PROCESSING.
+    await session.withTransaction(async () => {
+      // Only have a single replication stream with PROCESSING.
       await this.db.sync_rules.updateMany(
         {
           state: storage.SyncRuleState.PROCESSING
         },
-        { $set: { state: storage.SyncRuleState.STOP } }
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP),
+        { session }
       );
 
       const id_doc = await this.db.op_id_sequence.findOneAndUpdate(
@@ -180,14 +274,15 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
         },
         {
           upsert: true,
-          returnDocument: 'after'
+          returnDocument: 'after',
+          session
         }
       );
 
       const id = Number(id_doc!.op_id);
       const slot_name = generateSlotName(this.slot_name_prefix, id);
 
-      const doc: SyncRuleDocument = {
+      const doc: SyncRuleDocumentV1 = {
         _id: id,
         storage_version: storageVersion,
         content: options.config.yaml,
@@ -205,43 +300,68 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
         last_fatal_error_ts: null,
         last_keepalive_ts: null
       };
-      await this.db.sync_rules.insertOne(doc);
+
+      await this.db.sync_rules.insertOne(doc, { session });
       await this.db.notifyCheckpoint();
-      rules = new MongoPersistedSyncRulesContent(this.db, doc);
+      rules = new MongoPersistedSyncRulesContentV1(this.db, doc);
       if (options.lock) {
-        const lock = await rules.lock();
+        // The lock is persisted on rules.current_lock
+        await rules.lock(session);
       }
     });
 
     return rules!;
   }
 
-  async getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
+  async getActiveSyncRulesContent(): Promise<
+    MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null
+  > {
     const doc = await this.db.sync_rules.findOne(
       {
         state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
       },
       { sort: { _id: -1 }, limit: 1 }
     );
+
+    return this.getSyncRulesContent(doc, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]);
+  }
+
+  private async getSyncRulesContent(doc: SyncRuleDocumentBase | null, stateFilter: storage.SyncRuleState[]) {
     if (doc == null) {
       return null;
     }
+    const storageConfig = getMongoStorageConfig(doc.storage_version ?? LEGACY_STORAGE_VERSION);
+
+    if (storageConfig.incrementalReprocessing) {
+      const v3 = doc as ReplicationStreamDocumentV3;
+      const active = v3.sync_configs.find((c) => stateFilter.includes(c.state));
+      if (active == null) {
+        return null;
+      }
 
-    return new MongoPersistedSyncRulesContent(this.db, doc);
+      // TODO: cache the config. It could specifically help for the main replication loop
+      // that checks for active replication streams.
+      // It is not a major bottleneck though, since it only runs once every couple of seconds at most.
+      const db = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3;
+      const syncConfigDoc = await db.syncConfigDefinitions.findOne({ _id: active._id });
+      if (syncConfigDoc == null) {
+        return null;
+      }
+      return new MongoPersistedSyncRulesContentV3(this.db, v3, syncConfigDoc);
+    }
+
+    return new MongoPersistedSyncRulesContentV1(this.db, doc as SyncRuleDocumentV1);
   }
 
-  async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
+  async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null> {
     const doc = await this.db.sync_rules.findOne(
       {
         state: storage.SyncRuleState.PROCESSING
       },
       { sort: { _id: -1 }, limit: 1 }
     );
-    if (doc == null) {
-      return null;
-    }
 
-    return new MongoPersistedSyncRulesContent(this.db, doc);
+    return this.getSyncRulesContent(doc, [storage.SyncRuleState.PROCESSING]);
   }
 
   async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
@@ -251,9 +371,13 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       })
       .toArray();
 
-    return docs.map((doc) => {
-      return new MongoPersistedSyncRulesContent(this.db, doc);
-    });
+    return (
+      await Promise.all(
+        docs.map((doc) => {
+          return this.getSyncRulesContent(doc, [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]);
+        })
+      )
+    ).filter((r) => r != null);
   }
 
   async getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
@@ -263,9 +387,13 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       })
       .toArray();
 
-    return docs.map((doc) => {
-      return new MongoPersistedSyncRulesContent(this.db, doc);
-    });
+    return (
+      await Promise.all(
+        docs.map((doc) => {
+          return this.getSyncRulesContent(doc, [storage.SyncRuleState.STOP]);
+        })
+      )
+    ).filter((d) => d != null);
   }
 
   async getActiveStorage(): Promise<MongoSyncBucketStorage | null> {
@@ -296,63 +424,90 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       }
     };
 
-    const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
-    if (active_sync_rules == null) {
-      return {
-        operations_size_bytes: 0,
-        parameters_size_bytes: 0,
-        replication_size_bytes: 0
-      };
-    }
-    const operations_aggregate = await this.db.bucket_data
+    // For now, we get storage metrics over all v1 and v3 collections.
+    // In the future, we may split these metrics to report separately for active replication streams versus processing streams.
 
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
-          }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+    const aggregateStaticCollection = async <T extends mongo.Document>(collection: mongo.Collection<T>) => {
+      // We check whether the collection exists before getting the statistics. This avoids repeated
+      // errors in the MongoDB logs if the collection hasn't been created yet.
+      const exists =
+        (await this.db.db.listCollections({ name: collection.collectionName }, { nameOnly: true }).toArray()).length >
+        0;
+      if (!exists) {
+        return [{ storageStats: { size: 0 } }];
+      }
 
-    const parameters_aggregate = await this.db.bucket_parameters
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
+      return collection
+        .aggregate([
+          {
+            $collStats: {
+              storageStats: {}
+            }
           }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+        ])
+        .toArray()
+        .catch(ignoreNotExisting);
+    };
 
-    const v1_replication_aggregate = await this.db.current_data
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
-          }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+    const operations_aggregate = await aggregateStaticCollection(this.db.bucket_data);
+    const v3_operation_aggregates = await Promise.all(
+      (await this.db.listBucketDataCollectionsV3()).map((collection) =>
+        collection
+          .aggregate([
+            {
+              $collStats: {
+                storageStats: {}
+              }
+            }
+          ])
+          .toArray()
+          .catch(ignoreNotExisting)
+      )
+    );
 
-    const v3_replication_aggregate = await this.db.v3_current_data
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
-          }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+    const parameters_aggregate = await aggregateStaticCollection(this.db.bucket_parameters);
+
+    const v3_parameter_aggregates = await Promise.all(
+      (await this.db.listAllParameterIndexCollectionsV3()).map((collection) =>
+        collection
+          .aggregate([
+            {
+              $collStats: {
+                storageStats: {}
+              }
+            }
+          ])
+          .toArray()
+          .catch(ignoreNotExisting)
+      )
+    );
+
+    const v1_source_record_aggregate = await aggregateStaticCollection(this.db.current_data);
+
+    const source_record_aggregates = await Promise.all(
+      (await this.db.listAllSourceRecordCollectionsV3()).map((collection) =>
+        collection
+          .aggregate([
+            {
+              $collStats: {
+                storageStats: {}
+              }
+            }
+          ])
+          .toArray()
+          .catch(ignoreNotExisting)
+      )
+    );
     return {
-      operations_size_bytes: Number(operations_aggregate[0].storageStats.size),
-      parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size),
+      operations_size_bytes:
+        Number(operations_aggregate[0].storageStats.size) +
+        v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
+      parameters_size_bytes:
+        Number(parameters_aggregate[0].storageStats.size) +
+        v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
       replication_size_bytes:
-        Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size)
+        Number(v1_source_record_aggregate[0]?.storageStats?.size ?? 0) +
+        source_record_aggregates.reduce((total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), 0)
     };
   }
 

package/src/storage/implementation/BucketDefinitionMapping.ts

@@ -0,0 +1,75 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  BucketDataSource,
+  BucketDefinitionId,
+  ParameterIndexId,
+  ParameterIndexLookupCreator,
+  SyncConfigWithErrors
+} from '@powersync/service-sync-rules';
+import { SyncConfigDefinition } from '../storage-index.js';
+
+export class BucketDefinitionMapping {
+  static fromSyncConfig(doc: Pick<SyncConfigDefinition, 'rule_mapping'>): BucketDefinitionMapping {
+    return new BucketDefinitionMapping(doc.rule_mapping?.definitions ?? {}, doc.rule_mapping?.parameter_indexes ?? {});
+  }
+
+  static fromParsedSyncRules(syncRules: SyncConfigWithErrors): BucketDefinitionMapping {
+    const definitionNames = syncRules.config.bucketDataSources.map((source) => source.uniqueName).sort();
+    const parameterKeys = syncRules.config.bucketParameterLookupSources
+      .map((source) => `${source.sourceId.lookupName}#${source.sourceId.queryId}`)
+      .sort();
+
+    const definitions: Record<string, BucketDefinitionId> = {};
+    const parameterLookups: Record<string, ParameterIndexId> = {};
+
+    for (const [index, uniqueName] of definitionNames.entries()) {
+      definitions[uniqueName] = (index + 1).toString(16);
+    }
+    for (const [index, key] of parameterKeys.entries()) {
+      parameterLookups[key] = (index + 1).toString(16);
+    }
+
+    return new BucketDefinitionMapping(definitions, parameterLookups);
+  }
+
+  constructor(
+    private definitions: Record<string, BucketDefinitionId> = {},
+    private parameterLookupMapping: Record<string, ParameterIndexId> = {}
+  ) {}
+
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId {
+    const defId = this.definitions[source.uniqueName];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for bucket source ${source.uniqueName}`);
+    }
+    return defId;
+  }
+
+  allBucketDefinitionIds(): BucketDefinitionId[] {
+    return Object.values(this.definitions);
+  }
+
+  allParameterIndexIds(): ParameterIndexId[] {
+    return Object.values(this.parameterLookupMapping);
+  }
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId {
+    const key = this.parameterLookupKey(source.sourceId.lookupName, source.sourceId.queryId);
+    const defId = this.parameterLookupMapping[key];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`);
+    }
+    return defId;
+  }
+
+  private parameterLookupKey(lookupName: string, queryId: string) {
+    return `${lookupName}#${queryId}`;
+  }
+
+  serialize(): SyncConfigDefinition['rule_mapping'] {
+    return {
+      definitions: { ...this.definitions },
+      parameter_indexes: { ...this.parameterLookupMapping }
+    };
+  }
+}

package/src/storage/implementation/CheckpointState.ts

@@ -0,0 +1,59 @@
+export interface CheckpointStateInput {
+  lsn: string;
+  snapshotDone: boolean;
+  lastCheckpointLsn: string | null;
+  noCheckpointBefore: string | null;
+  keepaliveOp: bigint | null;
+  lastCheckpoint: bigint | null;
+  persistedOp: bigint | null;
+  createEmptyCheckpoints: boolean;
+}
+
+export interface CheckpointStateResult {
+  canCheckpoint: boolean;
+  checkpointBlocked: boolean;
+  checkpointCreated: boolean;
+  notEmpty: boolean;
+  newKeepaliveOp: bigint | null;
+  newLastCheckpoint: bigint | null;
+}
+
+function maxOpId(...values: (bigint | null | undefined)[]): bigint {
+  let max = 0n;
+  for (const value of values) {
+    if (value != null && value > max) {
+      max = value;
+    }
+  }
+  return max;
+}
+
+export function canCheckpointState(
+  lsn: string,
+  state: Pick<CheckpointStateInput, 'snapshotDone' | 'lastCheckpointLsn' | 'noCheckpointBefore'>
+): boolean {
+  return (
+    state.snapshotDone === true &&
+    (state.lastCheckpointLsn == null || state.lastCheckpointLsn <= lsn) &&
+    (state.noCheckpointBefore == null || state.noCheckpointBefore <= lsn)
+  );
+}
+
+export function calculateCheckpointState(input: CheckpointStateInput): CheckpointStateResult {
+  const canCheckpoint = canCheckpointState(input.lsn, input);
+  const newKeepaliveOp = canCheckpoint ? null : maxOpId(input.keepaliveOp, input.persistedOp);
+  const newLastCheckpoint = canCheckpoint
+    ? maxOpId(input.lastCheckpoint, input.persistedOp, input.keepaliveOp)
+    : input.lastCheckpoint;
+  const notEmpty =
+    input.createEmptyCheckpoints || input.keepaliveOp !== newKeepaliveOp || input.lastCheckpoint !== newLastCheckpoint;
+
+  return {
+    canCheckpoint,
+    checkpointBlocked: !canCheckpoint,
+    checkpointCreated: canCheckpoint && notEmpty,
+    notEmpty,
+    newKeepaliveOp,
+    newLastCheckpoint
+  };
+}