@powersync/service-module-mongodb-storage 0.12.13 → 0.12.14

package/src/storage/implementation/MongoBucketBatch.ts

@@ -82,6 +82,7 @@ export class MongoBucketBatch
   private batch: OperationBatch | null = null;
   private write_checkpoint_batch: storage.CustomWriteCheckpointOptions[] = [];
   private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
+  private clearedError = false;
 
   /**
    * Last LSN received associated with a checkpoint.
@@ -146,6 +147,10 @@ export class MongoBucketBatch
     return this.last_checkpoint_lsn;
   }
 
+  get noCheckpointBeforeLsn() {
+    return this.no_checkpoint_before_lsn;
+  }
+
   async flush(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null> {
     let result: storage.FlushedResult | null = null;
     // One flush may be split over multiple transactions.
@@ -243,6 +248,8 @@ export class MongoBucketBatch
     let resumeBatch: OperationBatch | null = null;
     let transactionSize = 0;
 
+    let didFlush = false;
+
     // Now batch according to the sizes
     // This is a single batch if storeCurrentData == false
     for await (let b of batch.batched(sizes)) {
@@ -292,7 +299,8 @@ export class MongoBucketBatch
         if (persistedBatch!.shouldFlushTransaction()) {
           // Transaction is getting big.
           // Flush, and resume in a new transaction.
-          await persistedBatch!.flush(this.db, this.session, options);
+          const { flushedAny } = await persistedBatch!.flush(this.db, this.session, options);
+          didFlush ||= flushedAny;
           persistedBatch = null;
           // Computing our current progress is a little tricky here, since
           // we're stopping in the middle of a batch.
@@ -303,10 +311,15 @@ export class MongoBucketBatch
 
       if (persistedBatch) {
         transactionSize = persistedBatch.currentSize;
-        await persistedBatch.flush(this.db, this.session, options);
+        const { flushedAny } = await persistedBatch.flush(this.db, this.session, options);
+        didFlush ||= flushedAny;
       }
     }
 
+    if (didFlush) {
+      await this.clearError();
+    }
+
     return resumeBatch?.hasData() ? resumeBatch : null;
   }
 
@@ -714,6 +727,7 @@ export class MongoBucketBatch
       last_keepalive_ts: now,
       snapshot_done: true,
       last_fatal_error: null,
+      last_fatal_error_ts: null,
       keepalive_op: null
     };
 
@@ -848,6 +862,7 @@ export class MongoBucketBatch
           last_checkpoint_lsn: lsn,
           snapshot_done: true,
           last_fatal_error: null,
+          last_fatal_error_ts: null,
           last_keepalive_ts: new Date()
         },
         $unset: { snapshot_lsn: 1 }
@@ -1075,6 +1090,26 @@ export class MongoBucketBatch
     });
   }
 
+  protected async clearError(): Promise<void> {
+    // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
+    if (this.clearedError) {
+      return;
+    }
+
+    await this.db.sync_rules.updateOne(
+      {
+        _id: this.group_id
+      },
+      {
+        $set: {
+          last_fatal_error: null,
+          last_fatal_error_ts: null
+        }
+      }
+    );
+    this.clearedError = true;
+  }
+
   /**
    * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
    */

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -13,6 +13,7 @@ export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRule
   public readonly sync_rules_content: string;
   public readonly last_checkpoint_lsn: string | null;
   public readonly last_fatal_error: string | null;
+  public readonly last_fatal_error_ts: Date | null;
   public readonly last_keepalive_ts: Date | null;
   public readonly last_checkpoint_ts: Date | null;
   public readonly active: boolean;
@@ -29,6 +30,7 @@ export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRule
     // Handle legacy values
     this.slot_name = doc.slot_name ?? `powersync_${this.id}`;
     this.last_fatal_error = doc.last_fatal_error;
+    this.last_fatal_error_ts = doc.last_fatal_error_ts;
     this.last_checkpoint_ts = doc.last_checkpoint_ts;
     this.last_keepalive_ts = doc.last_keepalive_ts;
     this.active = doc.state == 'ACTIVE';

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -39,7 +39,6 @@ import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
 
-
 export interface MongoSyncBucketStorageOptions {
   checksumOptions?: MongoChecksumOptions;
 }
@@ -648,11 +647,11 @@ export class MongoSyncBucketStorage
       },
       {
         $set: {
-          last_fatal_error: message
+          last_fatal_error: message,
+          last_fatal_error_ts: new Date()
         }
       }
     );
-    await this.db.notifyCheckpoint();
   }
 
   async compact(options?: storage.CompactOptions) {

package/src/storage/implementation/PersistedBatch.ts

@@ -351,12 +351,21 @@ export class PersistedBatch {
       }
     }
 
+    const stats = {
+      bucketDataCount: this.bucketData.length,
+      parameterDataCount: this.bucketParameters.length,
+      currentDataCount: this.currentData.length,
+      flushedAny: flushedSomething
+    };
+
     this.bucketData = [];
     this.bucketParameters = [];
     this.currentData = [];
     this.bucketStates.clear();
     this.currentSize = 0;
     this.debugLastOpId = null;
+
+    return stats;
   }
 
   private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocument>[] {

package/src/storage/implementation/models.ts

@@ -196,6 +196,8 @@ export interface SyncRuleDocument {
    */
   last_fatal_error: string | null;
 
+  last_fatal_error_ts: Date | null;
+
   content: string;
 
   lock?: {