@powersync/service-module-mongodb-storage 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts

@@ -0,0 +1,80 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { DO_NOT_LOG } from '@powersync/lib-services-framework';
+import { PowerSyncMongo } from '../db.js';
+import { CommonSourceTableDocument, StorageConfig } from '../models.js';
+
+export abstract class BaseVersionedPowerSyncMongo {
+  readonly client: mongo.MongoClient;
+  readonly db: mongo.Db;
+  readonly storageConfig: StorageConfig;
+  [DO_NOT_LOG] = true;
+
+  constructor(
+    protected readonly upstream: PowerSyncMongo,
+    storageConfig: StorageConfig
+  ) {
+    this.client = upstream.client;
+    this.db = upstream.db;
+    this.storageConfig = storageConfig;
+  }
+
+  get bucket_data() {
+    return this.upstream.bucket_data;
+  }
+
+  get op_id_sequence() {
+    return this.upstream.op_id_sequence;
+  }
+
+  get sync_rules() {
+    return this.upstream.sync_rules;
+  }
+
+  get custom_write_checkpoints() {
+    return this.upstream.custom_write_checkpoints;
+  }
+
+  get write_checkpoints() {
+    return this.upstream.write_checkpoints;
+  }
+
+  get instance() {
+    return this.upstream.instance;
+  }
+
+  get locks() {
+    return this.upstream.locks;
+  }
+
+  get checkpoint_events() {
+    return this.upstream.checkpoint_events;
+  }
+
+  get connection_report_events() {
+    return this.upstream.connection_report_events;
+  }
+
+  notifyCheckpoint() {
+    return this.upstream.notifyCheckpoint();
+  }
+
+  protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
+    return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
+  }
+
+  protected sourceTableCollectionName(replicationStreamId: number) {
+    return this.upstream.sourceTableCollectionName(replicationStreamId);
+  }
+
+  protected async listCollectionsByPrefix<T extends mongo.Document>(prefix: string): Promise<mongo.Collection<T>[]> {
+    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<T>(collection.name));
+  }
+
+  abstract commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument>;
+
+  abstract initializeStreamStorage(replicationStreamId: number): Promise<void>;
+}

package/src/storage/implementation/createMongoSyncBucketStorage.ts

@@ -0,0 +1,25 @@
+import { storage } from '@powersync/service-core';
+import { MongoBucketStorage } from '../MongoBucketStorage.js';
+import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
+import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
+import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
+
+export { MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
+
+export type { MongoSyncBucketStorage };
+
+export function createMongoSyncBucketStorage(
+  factory: MongoBucketStorage,
+  group_id: number,
+  sync_rules: MongoPersistedSyncRulesContent,
+  slot_name: string,
+  writeCheckpointMode: storage.WriteCheckpointMode | undefined,
+  options: MongoSyncBucketStorageOptions
+): MongoSyncBucketStorage {
+  if (sync_rules.getStorageConfig().incrementalReprocessing) {
+    return new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  }
+
+  return new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+}

package/src/storage/implementation/db.ts

@@ -2,21 +2,30 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
 
+import { DO_NOT_LOG } from '@powersync/lib-services-framework';
 import { MongoStorageConfig } from '../../types/types.js';
+import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBase.js';
 import {
-  BucketDataDocument,
-  BucketParameterDocument,
-  BucketStateDocument,
   CheckpointEventDocument,
   ClientConnectionDocument,
-  CurrentDataDocument,
+  CommonSourceTableDocument,
   CustomWriteCheckpointDocument,
   IdSequenceDocument,
   InstanceDocument,
   SourceTableDocument,
+  StorageConfig,
   SyncRuleDocument,
   WriteCheckpointDocument
 } from './models.js';
+import {
+  BucketDataDocumentV1,
+  BucketParameterDocument,
+  BucketStateDocumentV1,
+  CurrentDataDocument
+} from './v1/models.js';
+import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
+import { BucketDataDocumentV3 } from './v3/models.js';
+import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -26,8 +35,10 @@ export interface PowerSyncMongoOptions {
 }
 
 export class PowerSyncMongo {
+  [DO_NOT_LOG] = true;
+
   readonly current_data: mongo.Collection<CurrentDataDocument>;
-  readonly bucket_data: mongo.Collection<BucketDataDocument>;
+  readonly bucket_data: mongo.Collection<BucketDataDocumentV1>;
   readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
   readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
   readonly sync_rules: mongo.Collection<SyncRuleDocument>;
@@ -36,7 +47,7 @@ export class PowerSyncMongo {
   readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
   readonly instance: mongo.Collection<InstanceDocument>;
   readonly locks: mongo.Collection<lib_mongo.locks.Lock>;
-  readonly bucket_state: mongo.Collection<BucketStateDocument>;
+  readonly bucket_state: mongo.Collection<BucketStateDocumentV1>;
   readonly checkpoint_events: mongo.Collection<CheckpointEventDocument>;
   readonly connection_report_events: mongo.Collection<ClientConnectionDocument>;
 
@@ -51,7 +62,7 @@ export class PowerSyncMongo {
     });
     this.db = db;
 
-    this.current_data = db.collection<CurrentDataDocument>('current_data');
+    this.current_data = db.collection('current_data');
     this.bucket_data = db.collection('bucket_data');
     this.bucket_parameters = db.collection('bucket_parameters');
     this.op_id_sequence = db.collection('op_id_sequence');
@@ -66,15 +77,106 @@ export class PowerSyncMongo {
     this.connection_report_events = this.db.collection('connection_report_events');
   }
 
+  versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo {
+    if (storageConfig.incrementalReprocessing) {
+      return new VersionedPowerSyncMongoV3(this, storageConfig);
+    }
+
+    return new VersionedPowerSyncMongoV1(this, storageConfig);
+  }
+
+  /**
+   * Not safe for user-provided prefix - only for hardcoded values.
+   */
+  async listBucketDataCollectionsV3(groupId?: number): Promise<mongo.Collection<BucketDataDocumentV3>[]> {
+    const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
+    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<BucketDataDocumentV3>(collection.name));
+  }
+
+  /**
+   * Not safe for user-provided prefix - only for hardcoded values.
+   */
+  private async collectionsByPrefix(prefix: string): Promise<mongo.Collection<never>[]> {
+    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<never>(collection.name));
+  }
+
+  /**
+   * List all parameter index collections across all replication streams.
+   *
+   * Primarily used to clear the db.
+   */
+  async listAllParameterIndexCollectionsV3(): Promise<mongo.Collection<never>[]> {
+    return this.collectionsByPrefix(`parameter_index_`);
+  }
+
+  /**
+   * List all parameter index collections across all replication streams.
+   *
+   * Primarily used to clear the db.
+   */
+  async listAllSourceRecordCollectionsV3(): Promise<mongo.Collection<never>[]> {
+    return this.collectionsByPrefix(`source_records_`);
+  }
+
+  async listAllBucketStateCollectionsV3(): Promise<mongo.Collection<never>[]> {
+    return this.collectionsByPrefix(`bucket_state_`);
+  }
+
+  sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
+    return `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`;
+  }
+
+  sourceTableCollectionName(replicationStreamId: number) {
+    return `source_table_${replicationStreamId}`;
+  }
+
+  async listSourceTableCollections(
+    replicationStreamId?: number
+  ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
+    const filter =
+      replicationStreamId == null
+        ? { name: new RegExp('^source_table_') }
+        : { name: this.sourceTableCollectionName(replicationStreamId) };
+    const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId);
+    const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => this.db.collection<CommonSourceTableDocument>(collection.name));
+  }
+
   /**
    * Clear all collections.
    */
   async clear() {
     await this.current_data.deleteMany({});
+    for (const collection of await this.listAllSourceRecordCollectionsV3()) {
+      await collection.drop();
+    }
     await this.bucket_data.deleteMany({});
+    for (const collection of await this.listBucketDataCollectionsV3()) {
+      await collection.drop();
+    }
     await this.bucket_parameters.deleteMany({});
+    for (const collection of await this.listAllParameterIndexCollectionsV3()) {
+      await collection.drop();
+    }
+    for (const collection of await this.listAllBucketStateCollectionsV3()) {
+      await collection.drop();
+    }
     await this.op_id_sequence.deleteMany({});
     await this.sync_rules.deleteMany({});
+    for (const collection of await this.listSourceTableCollections()) {
+      await collection.drop();
+    }
     await this.source_tables.deleteMany({});
     await this.write_checkpoints.deleteMany({});
     await this.instance.deleteOne({});
@@ -93,7 +195,7 @@ export class PowerSyncMongo {
   }
 
   /**
-   * Call this after every checkpoint or sync rules status update. Rather call too often than too rarely.
+   * Call this after every checkpoint or replication stream status update. Rather call too often than too rarely.
    *
    * This is used in a similar way to the Postgres NOTIFY functionality.
    */
@@ -173,6 +275,11 @@ export class PowerSyncMongo {
   }
 }
 
+/**
+ * This is similar to PowerSyncMongo, but blocks access to certain collections based on the storage version.
+ */
+export type VersionedPowerSyncMongo = BaseVersionedPowerSyncMongo;
+
 export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) {
   return new PowerSyncMongo(
     lib_mongo.createMongoClient(config, {

package/src/storage/implementation/models.ts

@@ -1,7 +1,11 @@
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
 import { event_types } from '@powersync/service-types';
+import * as bson from 'bson';
+import { ParameterIndexId } from './BucketDefinitionMapping.js';
+import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
+import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -21,37 +25,53 @@ export interface SourceKey {
   k: ReplicaId;
 }
 
+export interface SourceTableKey {
+  /** source table id */
+  t: bson.ObjectId;
+  /** source key */
+  k: ReplicaId;
+}
+
 export interface BucketDataKey {
-  /** group_id */
-  g: number;
   /** bucket name */
   b: string;
   /** op_id */
   o: bigint;
 }
 
-export interface CurrentDataDocument {
-  _id: SourceKey;
-  data: bson.Binary;
-  buckets: CurrentBucket[];
-  lookups: bson.Binary[];
-}
-
 export interface CurrentBucket {
   bucket: string;
   table: string;
   id: string;
 }
 
-export interface BucketParameterDocument {
+export interface BucketParameterDocumentBase<TKey> {
   _id: bigint;
-  key: SourceKey;
+  key: TKey;
   lookup: bson.Binary;
   bucket_parameters: Record<string, SqliteJsonValue>[];
 }
 
-export interface BucketDataDocument {
-  _id: BucketDataKey;
+export interface TaggedBucketParameterDocument extends BucketParameterDocumentBase<SourceKey | SourceTableKey> {
+  index: ParameterIndexId;
+}
+
+export function bucketParameterDocumentToTagged<TKey extends SourceKey | SourceTableKey>(
+  document: BucketParameterDocumentBase<TKey>,
+  index: ParameterIndexId
+): TaggedBucketParameterDocument {
+  return {
+    ...document,
+    index
+  };
+}
+
+export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+
+/**
+ * Common properties for storage V1, storage V3 and in-memory BucketDataDoc.
+ */
+export interface BucketDataProperties {
   op: OpType;
   source_table?: bson.ObjectId;
   source_key?: ReplicaId;
@@ -62,11 +82,22 @@ export interface BucketDataDocument {
   target_op?: bigint | null;
 }
 
-export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+export interface BucketDataDocumentBase extends BucketDataProperties {
+  _id: { b: string };
+}
+
+/**
+ * Internal-only tag used for v1 bucket_data rows before they are converted to the v1 on-disk shape.
+ */
+export const LEGACY_BUCKET_DATA_DEFINITION_ID = '0';
+
+/**
+ * Internal-only tag used for v1 bucket_parameters rows before they are converted to the v1 on-disk shape.
+ */
+export const LEGACY_BUCKET_PARAMETER_INDEX_ID = '0';
 
 export interface SourceTableDocument {
   _id: bson.ObjectId;
-  group_id: number;
   connection_id: number;
   relation_id: number | string | undefined;
   schema_name: string;
@@ -86,20 +117,21 @@ export interface SourceTableDocumentSnapshotStatus {
 /**
  * Record the state of each bucket.
  *
- * Right now, this is just used to track when buckets are updated, for efficient incremental sync.
- * In the future, this could be used to track operation counts, both for diagnostic purposes, and for
- * determining when a compact and/or defragment could be beneficial.
+ * The primary use case is to track when buckets are updated, for efficient incremental sync.
+ *
+ * The secondary use case is to track operation counts to determine whether or not a bucket should be compacted.
  *
- * Note: There is currently no migration to populate this collection from existing data - it is only
+ * Note: For storage V1, there is no migration to populate this collection from existing data - it is only
  * populated by new updates.
+ *
+ * For storage V3, these will always be present.
  */
-export interface BucketStateDocument {
+export interface BucketStateDocumentBase {
   _id: {
-    g: number;
     b: string;
   };
   /**
-   * Important: There is an unique index on {'_id.g': 1, last_op: 1}.
+   * Important: There is an unique index on last_op per logical stream.
    * That means the last_op must match an actual op in the bucket, and not the commit checkpoint.
    */
   last_op: bigint;
@@ -111,12 +143,12 @@ export interface BucketStateDocument {
     op_id: InternalOpId;
     count: number;
     checksum: bigint;
-    bytes: number | null;
+    bytes: number | bigint | null;
   };
 
   estimate_since_compact?: {
     count: number;
-    bytes: number;
+    bytes: number | bigint;
   };
 }
 
@@ -201,6 +233,20 @@ export interface SyncRuleDocument {
   content: string;
   serialized_plan?: SerializedSyncPlan | null;
 
+  /**
+   * Required for V3+ storage.
+   */
+  rule_mapping?: {
+    /**
+     * Map of uniqueName -> id, unique per replication stream.
+     */
+    definitions: Record<string, string>;
+    /**
+     * Map of (lookupName, queryId) -> id, unique per replication stream.
+     */
+    parameter_indexes: Record<string, string>;
+  };
+
   lock?: {
     id: string;
     expires_at: Date;
@@ -217,17 +263,26 @@ export interface StorageConfig extends storage.StorageVersionConfig {
    * a Long before summing.
    */
   longChecksums: boolean;
+  /**
+   * Enables v3 MongoDB storage behavior used for incremental reprocessing.
+   */
+  incrementalReprocessing: boolean;
 }
 
 const LONG_CHECKSUMS_STORAGE_VERSION = 2;
+const INCREMENTAL_REPROCESSING_STORAGE_VERSION = storage.STORAGE_VERSION_3;
 
-export function getMongoStorageConfig(storageVersion: number): StorageConfig | undefined {
+export function getMongoStorageConfig(storageVersion: number): StorageConfig {
   const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
   if (baseConfig == null) {
-    return undefined;
+    throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${storageVersion}`);
   }
 
-  return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION };
+  return {
+    ...baseConfig,
+    longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION,
+    incrementalReprocessing: storageVersion >= INCREMENTAL_REPROCESSING_STORAGE_VERSION
+  };
 }
 
 export interface CheckpointEventDocument {
@@ -272,3 +327,8 @@ export interface InstanceDocument {
 }
 
 export interface ClientConnectionDocument extends event_types.ClientConnection {}
+
+export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id'];
+export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3;
+export type CommonCurrentLookup = bson.Binary | RecordedLookupV3;
+export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3;

package/src/storage/implementation/v1/MongoBucketBatchV1.ts

@@ -0,0 +1,32 @@
+import { SourceTable } from '@powersync/service-core';
+import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { PersistedBatch } from '../common/PersistedBatch.js';
+import { SourceRecordStore } from '../common/SourceRecordStore.js';
+import { PersistedBatchV1 } from './PersistedBatchV1.js';
+import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+
+export class MongoBucketBatchV1 extends MongoBucketBatch {
+  declare public readonly db: VersionedPowerSyncMongoV1;
+
+  private readonly store: SourceRecordStore;
+
+  constructor(options: MongoBucketBatchOptions) {
+    super(options);
+    this.store = new SourceRecordStoreV1(this.db, this.group_id);
+  }
+
+  protected createPersistedBatch(writtenSize: number): PersistedBatch {
+    return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, {
+      logger: this.logger
+    });
+  }
+
+  protected get sourceRecordStore(): SourceRecordStore {
+    return this.store;
+  }
+
+  protected async cleanupDroppedSourceTables(_tables: SourceTable[]) {
+    // No-op for V1: source records live in a shared collection.
+  }
+}

package/src/storage/implementation/v1/MongoChecksumsV1.ts

@@ -0,0 +1,75 @@
+import {
+  bson,
+  BucketChecksum,
+  FetchPartialBucketChecksum,
+  InternalOpId,
+  PartialChecksumMap
+} from '@powersync/service-core';
+import { FetchPartialBucketChecksumByBucket, MongoChecksums } from '../MongoChecksums.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+
+export class MongoChecksumsV1 extends MongoChecksums {
+  declare protected readonly db: VersionedPowerSyncMongoV1;
+
+  async computePartialChecksumsDirectByBucket(
+    batch: FetchPartialBucketChecksumByBucket[]
+  ): Promise<PartialChecksumMap> {
+    return this.computePartialChecksumsForCollection(batch, this.db.bucketDataV1, (request) => ({
+      _id: {
+        $gt: {
+          g: this.group_id,
+          b: request.bucket,
+          o: request.start ?? new bson.MinKey()
+        },
+        $lte: {
+          g: this.group_id,
+          b: request.bucket,
+          o: request.end
+        }
+      }
+    }));
+  }
+
+  protected async fetchPreStates(
+    batch: FetchPartialBucketChecksum[]
+  ): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>> {
+    const preFilters = batch
+      .filter((request) => request.start == null)
+      .map((request) => ({
+        _id: {
+          g: this.group_id,
+          b: request.bucket
+        },
+        'compacted_state.op_id': { $exists: true, $lte: request.end }
+      }));
+
+    const preStates = new Map<string, { opId: InternalOpId; checksum: BucketChecksum }>();
+    if (preFilters.length == 0) {
+      return preStates;
+    }
+
+    const states = await this.db.bucketStateV1
+      .find({
+        $or: preFilters
+      })
+      .toArray();
+
+    for (const state of states) {
+      const compactedState = state.compacted_state!;
+      preStates.set(state._id.b, {
+        opId: compactedState.op_id,
+        checksum: {
+          bucket: state._id.b,
+          checksum: Number(compactedState.checksum),
+          count: compactedState.count
+        }
+      });
+    }
+
+    return preStates;
+  }
+
+  protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
+    return this.computePartialChecksumsDirectByBucket(batch);
+  }
+}