@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts

@@ -67,7 +67,7 @@ export abstract class BaseVersionedPowerSyncMongo {
   }
 
   protected async listCollectionsByPrefix<T extends mongo.Document>(prefix: string): Promise<mongo.Collection<T>[]> {
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+    const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
 
     return collections
       .filter((collection) => collection.name.startsWith(prefix))

package/src/storage/implementation/createMongoSyncBucketStorage.ts

@@ -1,6 +1,6 @@
 import { storage } from '@powersync/service-core';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
 import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
 import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
 import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
@@ -11,15 +11,29 @@ export type { MongoSyncBucketStorage };
 
 export function createMongoSyncBucketStorage(
   factory: MongoBucketStorage,
-  group_id: number,
-  sync_rules: MongoPersistedSyncRulesContentV1,
-  slot_name: string,
+  replicationStreamId: number,
+  replicationStream: MongoPersistedReplicationStream,
+  replicationStreamName: string,
   writeCheckpointMode: storage.WriteCheckpointMode | undefined,
   options: MongoSyncBucketStorageOptions
 ): MongoSyncBucketStorage {
-  if (sync_rules.getStorageConfig().incrementalReprocessing) {
-    return new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  if (replicationStream.getStorageConfig().incrementalReprocessing) {
+    return new MongoSyncBucketStorageV3(
+      factory,
+      replicationStreamId,
+      replicationStream,
+      replicationStreamName,
+      writeCheckpointMode,
+      options
+    );
   }
 
-  return new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
+  return new MongoSyncBucketStorageV1(
+    factory,
+    replicationStreamId,
+    replicationStream,
+    replicationStreamName,
+    writeCheckpointMode,
+    options
+  );
 }

package/src/storage/implementation/db.ts

@@ -24,8 +24,9 @@ import {
   CurrentDataDocument
 } from './v1/models.js';
 import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
-import { BucketDataDocumentV3 } from './v3/models.js';
-import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
+import { BucketDataDocumentV3, CustomCheckpointRequestDocumentV3 } from './v3/models.js';
+import type { CustomCheckpointRequestCollectionSpecifier } from './v3/VersionedPowerSyncMongoV3.js';
+import { OBJECT_STORAGE_USAGE_COLLECTION, VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -81,7 +82,7 @@ export class PowerSyncMongo {
   versioned(storageConfig: StorageConfig & { incrementalReprocessing: false }): VersionedPowerSyncMongoV1;
   versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo;
   versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo {
-    if (storageConfig.incrementalReprocessing) {
+    if (storageConfig.version >= storage.STORAGE_VERSION_3) {
       return new VersionedPowerSyncMongoV3(this, storageConfig);
     }
 
@@ -93,22 +94,18 @@ export class PowerSyncMongo {
    */
   async listBucketDataCollectionsV3(groupId?: number): Promise<mongo.Collection<BucketDataDocumentV3>[]> {
     const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
-
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<BucketDataDocumentV3>(collection.name));
+    return this.collectionsByPrefix<BucketDataDocumentV3>(prefix);
   }
 
   /**
    * Not safe for user-provided prefix - only for hardcoded values.
    */
-  private async collectionsByPrefix(prefix: string): Promise<mongo.Collection<never>[]> {
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+  private async collectionsByPrefix<T extends mongo.Document = never>(prefix: string): Promise<mongo.Collection<T>[]> {
+    const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
 
     return collections
       .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<never>(collection.name));
+      .map((collection) => this.db.collection<T>(collection.name));
   }
 
   /**
@@ -141,19 +138,29 @@ export class PowerSyncMongo {
     return `source_table_${replicationStreamId}`;
   }
 
-  async listSourceTableCollections(
+  customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
+    return `custom_checkpoint_requests_${options.replicationStreamId}_${options.eventId}`;
+  }
+
+  async listCustomCheckpointRequestCollections(
     replicationStreamId?: number
-  ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
-    const filter =
+  ): Promise<mongo.Collection<CustomCheckpointRequestDocumentV3>[]> {
+    const prefix =
       replicationStreamId == null
-        ? { name: new RegExp('^source_table_') }
-        : { name: this.sourceTableCollectionName(replicationStreamId) };
-    const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId);
-    const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray();
+        ? 'custom_checkpoint_requests_'
+        : `custom_checkpoint_requests_${replicationStreamId}_`;
+    return this.collectionsByPrefix(prefix);
+  }
 
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => this.db.collection<CommonSourceTableDocument>(collection.name));
+  async listSourceTableCollections(
+    replicationStreamId?: number
+  ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
+    if (replicationStreamId == null) {
+      return this.collectionsByPrefix('source_table_');
+    }
+    const name = this.sourceTableCollectionName(replicationStreamId);
+    const collections = await this.db.listCollections({ name }, { nameOnly: true }).toArray();
+    return collections.map((c) => this.db.collection<CommonSourceTableDocument>(c.name));
   }
 
   /**
@@ -186,6 +193,10 @@ export class PowerSyncMongo {
     await this.locks.deleteMany({});
     await this.bucket_state.deleteMany({});
     await this.custom_write_checkpoints.deleteMany({});
+    await this.db.collection(OBJECT_STORAGE_USAGE_COLLECTION).deleteMany({});
+    for (const collection of await this.listCustomCheckpointRequestCollections()) {
+      await collection.drop();
+    }
   }
 
   /**
@@ -203,7 +214,14 @@ export class PowerSyncMongo {
    * This is used in a similar way to the Postgres NOTIFY functionality.
    */
   async notifyCheckpoint() {
-    await this.checkpoint_events.insertOne({} as any, { forceServerObjectId: true });
+    await this.checkpoint_events.insertOne({} as any, {
+      forceServerObjectId: true,
+      // No correctness issue if this write is missed / rolled back.
+      // It could cause some sync delays, that will resolve in at most a minute, but no consistency issues.
+      writeConcern: {
+        w: 1
+      }
+    });
   }
 
   /**

package/src/storage/implementation/models.ts

@@ -4,7 +4,7 @@ import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules
 import { event_types } from '@powersync/service-types';
 import * as bson from 'bson';
 import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
-import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js';
+import type { CurrentDataDocumentV3, RecordedLookupV3 } from './v3/models.js';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -74,10 +74,21 @@ export interface BucketDataProperties {
   op: OpType;
   source_table?: bson.ObjectId;
   source_key?: ReplicaId;
+  /**
+   * Stable identifier for the source record in the client protocol.
+   *
+   * This is derived from (source_table, source_key), so persistence is optional but preferred.
+   */
+  subkey?: string;
   table?: string;
   row_id?: string;
   checksum: bigint;
   data: string | null;
+  /**
+   * V1-only.
+   *
+   * V3 stores this on the BucketDataDocumentV3 instead of the individual ops.
+   */
   target_op?: bigint | null;
 }
 
@@ -105,6 +116,10 @@ export interface SourceTableDocument {
   replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
   snapshot_done: boolean | undefined;
   snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
+  /**
+   * Source-specific metadata. Absent for legacy records.
+   */
+  source_metadata?: storage.JsonValue;
 }
 
 export interface SourceTableDocumentSnapshotStatus {
@@ -206,9 +221,37 @@ export interface SyncRuleDocumentBase {
   } | null;
 
   storage_version?: number;
+
+  /** Optional operator-supplied label identifying the sync config version. */
+  version_label?: string | null;
+
+  /**
+   * Incremental parameter compaction state for the replication stream.
+   *
+   * Operation ids are allocated across all parameter indexes of a stream, so this is
+   * stream-level state, shared by all sync configs of the stream.
+   */
+  parameter_compaction?: {
+    /**
+     * The exclusive operation-id boundary through which every parameter index in this stream has
+     * been compacted.
+     */
+    compacted_before: InternalOpId;
+
+    /**
+     * Parameter entries below this boundary may no longer be available for checkpoint change
+     * detection, since compaction may have deleted them.
+     *
+     * This is advanced before the first delete of a compaction pass, while
+     * {@link compacted_before} is only advanced after every delete of the pass completed.
+     * A checkpoint transition starting below this boundary must conservatively invalidate all
+     * parameter buckets, since the individual changes may no longer be available.
+     */
+    checkpoint_changes_invalid_before?: InternalOpId;
+  };
 }
 
-export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint | null> {
+export interface SyncRuleCheckpointFields {
   /**
    * The last consistent checkpoint.
    *
@@ -225,16 +268,6 @@ export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint |
    * If set, no new checkpoints may be created < this value.
    */
   no_checkpoint_before: string | null;
-
-  /**
-   * Goes together with no_checkpoint_before.
-   *
-   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
-   * then the checkpoint must be equal to this keepalive_op.
-   *
-   * This is a string in V1, bigint in V3.
-   */
-  keepalive_op: TKeepaliveOp;
 }
 
 export interface StorageConfig extends storage.StorageVersionConfig {
@@ -283,6 +316,11 @@ export interface CustomWriteCheckpointDocument {
    * This is not unique - multiple write checkpoints can have the same op_id.
    */
   op_id?: InternalOpId;
+  /**
+   * Set when this checkpoint was created from a client-supplied checkpoint
+   * request rather than a persistent custom write checkpoint.
+   */
+  checkpoint_requested_at?: Date | null;
 }
 
 export interface WriteCheckpointDocument {
@@ -296,6 +334,12 @@ export interface WriteCheckpointDocument {
    * between two checkpoints.
    */
   processed_at_lsn: string | null;
+  /**
+   * Set when this checkpoint was created from a client-supplied checkpoint request
+   * rather than a server-generated id. A missing/null value means this is not a
+   * checkpoint request.
+   */
+  checkpoint_requested_at?: Date | null;
 }
 
 export interface InstanceDocument {
@@ -306,6 +350,6 @@ export interface InstanceDocument {
 export interface ClientConnectionDocument extends event_types.ClientConnection {}
 
 export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id'];
-export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3;
+export type CommonCurrentBucket = CurrentBucket;
 export type CommonCurrentLookup = bson.Binary | RecordedLookupV3;
-export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3;
+export type CommonSourceTableDocument = Pick<SourceTableDocumentV1, '_id' | 'snapshot_status'>;