@powersync/service-module-mongodb-storage 0.0.0-dev-20260203155513 → 0.0.0-dev-20260223080959

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -1,48 +1,41 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
 import { PowerSyncMongo } from './db.js';
-import { SyncRuleDocument } from './models.js';
-
-export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRulesContent {
-  public readonly slot_name: string;
-
-  public readonly id: number;
-  public readonly sync_rules_content: string;
-  public readonly last_checkpoint_lsn: string | null;
-  public readonly last_fatal_error: string | null;
-  public readonly last_fatal_error_ts: Date | null;
-  public readonly last_keepalive_ts: Date | null;
-  public readonly last_checkpoint_ts: Date | null;
-  public readonly active: boolean;
+import { getMongoStorageConfig, SyncRuleDocument } from './models.js';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 
+export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
   public current_lock: MongoSyncRulesLock | null = null;
 
   constructor(
     private db: PowerSyncMongo,
     doc: mongo.WithId<SyncRuleDocument>
   ) {
-    this.id = doc._id;
-    this.sync_rules_content = doc.content;
-    this.last_checkpoint_lsn = doc.last_checkpoint_lsn;
-    // Handle legacy values
-    this.slot_name = doc.slot_name ?? `powersync_${this.id}`;
-    this.last_fatal_error = doc.last_fatal_error;
-    this.last_fatal_error_ts = doc.last_fatal_error_ts;
-    this.last_checkpoint_ts = doc.last_checkpoint_ts;
-    this.last_keepalive_ts = doc.last_keepalive_ts;
-    this.active = doc.state == 'ACTIVE';
+    super({
+      id: doc._id,
+      sync_rules_content: doc.content,
+      last_checkpoint_lsn: doc.last_checkpoint_lsn,
+      // Handle legacy values
+      slot_name: doc.slot_name ?? `powersync_${doc._id}`,
+      last_fatal_error: doc.last_fatal_error,
+      last_fatal_error_ts: doc.last_fatal_error_ts,
+      last_checkpoint_ts: doc.last_checkpoint_ts,
+      last_keepalive_ts: doc.last_keepalive_ts,
+      active: doc.state == 'ACTIVE',
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION
+    });
   }
 
-  parsed(options: storage.ParseSyncRulesOptions) {
-    return new MongoPersistedSyncRules(
-      this.id,
-      SqlSyncRules.fromYaml(this.sync_rules_content, options),
-      this.last_checkpoint_lsn,
-      this.slot_name
-    );
+  getStorageConfig() {
+    const storageConfig = getMongoStorageConfig(this.storageVersion);
+    if (storageConfig == null) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
+      );
+    }
+    return storageConfig;
   }
 
   async lock() {

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -15,6 +15,7 @@ import {
   InternalOpId,
   internalToExternalOpId,
   maxLsn,
+  mergeAsyncIterables,
   PopulateChecksumCacheOptions,
   PopulateChecksumCacheResults,
   ProtocolOpId,
@@ -31,7 +32,14 @@ import * as timers from 'timers/promises';
 import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js';
+import {
+  BucketDataDocument,
+  BucketDataKey,
+  BucketStateDocument,
+  SourceKey,
+  SourceTableDocument,
+  StorageConfig
+} from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactor } from './MongoCompactor.js';
@@ -39,7 +47,8 @@ import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 
 export interface MongoSyncBucketStorageOptions {
-  checksumOptions?: MongoChecksumOptions;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  storageConfig: StorageConfig;
 }
 
 /**
@@ -68,12 +77,15 @@ export class MongoSyncBucketStorage
     public readonly group_id: number,
     private readonly sync_rules: storage.PersistedSyncRulesContent,
     public readonly slot_name: string,
-    writeCheckpointMode?: storage.WriteCheckpointMode,
-    options?: MongoSyncBucketStorageOptions
+    writeCheckpointMode: storage.WriteCheckpointMode | undefined,
+    options: MongoSyncBucketStorageOptions
   ) {
     super();
     this.db = factory.db;
-    this.checksums = new MongoChecksums(this.db, this.group_id, options?.checksumOptions);
+    this.checksums = new MongoChecksums(this.db, this.group_id, {
+      ...options.checksumOptions,
+      storageConfig: options?.storageConfig
+    });
     this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
       db: this.db,
       mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
@@ -694,53 +706,39 @@ export class MongoSyncBucketStorage
    * Instance-wide watch on the latest available checkpoint (op_id + lsn).
    */
   private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ReplicationCheckpoint> {
-    const stream = this.checkpointChangesStream(signal);
-
     if (signal.aborted) {
       return;
     }
 
+    // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS) before we get another checkpoint,
+    // to avoid stale checkpoint snapshots. This is what checkpointTimeoutStream() is for.
+    // Essentially, even if there are no actual checkpoint changes, we want a new snapshotTime every minute or so,
+    // to ensure that any new clients connecting will get a valid snapshotTime.
+    const stream = mergeAsyncIterables(
+      [this.checkpointChangesStream(signal), this.checkpointTimeoutStream(signal)],
+      signal
+    );
+
     // We only watch changes to the active sync rules.
     // If it changes to inactive, we abort and restart with the new sync rules.
-    try {
-      while (true) {
-        // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS)
-        // before we get another checkpoint, to avoid stale checkpoint snapshots.
-        const timeout = timers
-          .setTimeout(CHECKPOINT_TIMEOUT_MS, { done: false }, { signal })
-          .catch(() => ({ done: true }));
-        try {
-          const result = await Promise.race([stream.next(), timeout]);
-          if (result.done) {
-            break;
-          }
-        } catch (e) {
-          if (e.name == 'AbortError') {
-            break;
-          }
-          throw e;
-        }
-
-        if (signal.aborted) {
-          // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway
-          break;
-        }
-
-        const op = await this.getCheckpointInternal();
-        if (op == null) {
-          // Sync rules have changed - abort and restart.
-          // We do a soft close of the stream here - no error
-          break;
-        }
+    for await (const _ of stream) {
+      if (signal.aborted) {
+        // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway
+        break;
+      }
 
-        // Previously, we only yielded when the checkpoint or lsn changed.
-        // However, we always want to use the latest snapshotTime, so we skip that filtering here.
-        // That filtering could be added in the per-user streams if needed, but in general the capped collection
-        // should already only contain useful changes in most cases.
-        yield op;
+      const op = await this.getCheckpointInternal();
+      if (op == null) {
+        // Sync rules have changed - abort and restart.
+        // We do a soft close of the stream here - no error
+        break;
       }
-    } finally {
-      await stream.return(null);
+
+      // Previously, we only yielded when the checkpoint or lsn changed.
+      // However, we always want to use the latest snapshotTime, so we skip that filtering here.
+      // That filtering could be added in the per-user streams if needed, but in general the capped collection
+      // should already only contain useful changes in most cases.
+      yield op;
     }
   }
 
@@ -900,6 +898,24 @@ export class MongoSyncBucketStorage
     }
   }
 
+  private async *checkpointTimeoutStream(signal: AbortSignal): AsyncGenerator<void> {
+    while (!signal.aborted) {
+      try {
+        await timers.setTimeout(CHECKPOINT_TIMEOUT_MS, undefined, { signal });
+      } catch (e) {
+        if (e.name == 'AbortError') {
+          // This is how we typically abort this stream, when all listeners are done
+          return;
+        }
+        throw e;
+      }
+
+      if (!signal.aborted) {
+        yield;
+      }
+    }
+  }
+
   private async getDataBucketChanges(
     options: GetCheckpointChangesOptions
   ): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {

package/src/storage/implementation/db.ts

@@ -39,8 +39,6 @@ export class PowerSyncMongo {
   readonly bucket_state: mongo.Collection<BucketStateDocument>;
   readonly checkpoint_events: mongo.Collection<CheckpointEventDocument>;
   readonly connection_report_events: mongo.Collection<ClientConnectionDocument>;
-  readonly sync_report_events: mongo.Collection<any>;
-  readonly bucket_report_events: mongo.Collection<any>;
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
@@ -66,8 +64,6 @@ export class PowerSyncMongo {
     this.bucket_state = this.db.collection('bucket_state');
     this.checkpoint_events = this.db.collection('checkpoint_events');
     this.connection_report_events = this.db.collection('connection_report_events');
-    this.sync_report_events = this.db.collection('sync_report_events');
-    this.bucket_report_events = this.db.collection('bucket_report_events');
   }
 
   /**
@@ -149,34 +145,6 @@ export class PowerSyncMongo {
     await this.db.createCollection('connection_report_events');
   }
 
-  /**
-   * Only use in migrations and tests.
-   */
-  async createSyncReportingCollection() {
-    const existingCollections = await this.db
-      .listCollections({ name: 'sync_report_events' }, { nameOnly: false })
-      .toArray();
-    const collection = existingCollections[0];
-    if (collection != null) {
-      return;
-    }
-    await this.db.createCollection('sync_report_events');
-  }
-
-  /**
-   * Only use in migrations and tests.
-   */
-  async createBucketReportingCollection() {
-    const existingCollections = await this.db
-      .listCollections({ name: 'bucket_report_events' }, { nameOnly: false })
-      .toArray();
-    const collection = existingCollections[0];
-    if (collection != null) {
-      return;
-    }
-    await this.db.createCollection('bucket_report_events');
-  }
-
   /**
    * Only use in migrations and tests.
    */

package/src/storage/implementation/models.ts

@@ -204,6 +204,29 @@ export interface SyncRuleDocument {
     id: string;
     expires_at: Date;
   } | null;
+
+  storage_version?: number;
+}
+
+export interface StorageConfig extends storage.StorageVersionConfig {
+  /**
+   * When true, bucket_data.checksum is guaranteed to be persisted as a Long.
+   *
+   * When false, it could also have been persisted as an Int32 or Double, in which case it must be converted to
+   * a Long before summing.
+   */
+  longChecksums: boolean;
+}
+
+const LONG_CHECKSUMS_STORAGE_VERSION = 2;
+
+export function getMongoStorageConfig(storageVersion: number): StorageConfig | undefined {
+  const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
+  if (baseConfig == null) {
+    return undefined;
+  }
+
+  return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION };
 }
 
 export interface CheckpointEventDocument {

package/src/storage/storage-index.ts

@@ -2,7 +2,6 @@ export * from './implementation/db.js';
 export * from './implementation/models.js';
 export * from './implementation/MongoBucketBatch.js';
 export * from './implementation/MongoIdSequence.js';
-export * from './implementation/MongoPersistedSyncRules.js';
 export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncBucketStorage.js';

package/src/utils/test-utils.ts

@@ -1,14 +1,13 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { PowerSyncMongo } from '../storage/implementation/db.js';
 import { TestStorageOptions } from '@powersync/service-core';
+import { MongoBucketStorage, MongoBucketStorageOptions } from '../storage/MongoBucketStorage.js';
 import { MongoReportStorage } from '../storage/MongoReportStorage.js';
-import { MongoBucketStorage } from '../storage/MongoBucketStorage.js';
-import { MongoSyncBucketStorageOptions } from '../storage/implementation/MongoSyncBucketStorage.js';
+import { PowerSyncMongo } from '../storage/implementation/db.js';
 
 export type MongoTestStorageOptions = {
   url: string;
   isCI: boolean;
-  internalOptions?: MongoSyncBucketStorageOptions;
+  internalOptions?: MongoBucketStorageOptions;
 };
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {

package/src/utils/util.ts

@@ -142,6 +142,9 @@ export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
 
   const items = await findCursor.limit(limit).toArray();
   const count = items.length;
+  /** The returned total has been defaulted to 0 due to the overhead using documentCount from the mogo driver.
+   * cursor.count has been deprecated.
+   * */
   return {
     items,
     count,
@@ -150,39 +153,3 @@ export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
     more: !(count !== limit)
   };
 };
-
-export const createPaginatedSyncCheckpointQuery = async <T extends mongo.Document>(
-  query: mongo.Filter<T>,
-  collection: mongo.Collection<T>,
-  limit: number,
-  cursor?: string
-) => {
-  const createQuery = (cursor?: string) => {
-    if (!cursor) {
-      return query;
-    }
-    const date = { $lt: new Date(cursor), $gte: query.date.$gte };
-
-    return {
-      ...query,
-      ...date
-    } as mongo.Filter<T>;
-  };
-
-  const findCursor = collection.find(createQuery(cursor), {
-    sort: {
-      /** We are sorting by date at date descending to match cursor Postgres implementation */
-      date: -1
-    }
-  });
-
-  const items = await findCursor.limit(limit).toArray();
-  const count = items.length;
-  return {
-    items,
-    count,
-    /** Setting the cursor to the connected at date of the last item in the list */
-    cursor: count === limit ? items[items.length - 1].date.toISOString() : undefined,
-    more: !(count !== limit)
-  };
-};