@powersync/service-module-mongodb-storage 0.23.2 → 0.24.0

package/dist/storage/implementation/MongoBucketBatch.d.ts

@@ -4,18 +4,19 @@ import * as bson from 'bson';
 import { BaseObserver, Logger } from '@powersync/lib-services-framework';
 import { BucketDefinitionMapping, BucketStorageMarkRecordUnavailable, InternalOpId, PerformanceTracer, storage } from '@powersync/service-core';
 import { PersistedBatch } from './common/PersistedBatch.js';
-import { SourceRecordStore } from './common/SourceRecordStore.js';
+import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js';
 import type { VersionedPowerSyncMongo } from './db.js';
 import { SyncRuleDocumentBase } from './models.js';
+import { MongoIdSequence } from './MongoIdSequence.js';
 import { MongoOpIdAllocator } from './MongoOpIdAllocator.js';
 import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
 import { MongoWriteBatch } from './MongoWriteBatch.js';
+import { OperationBatch, RecordOperation } from './OperationBatch.js';
 import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
 export interface MongoBucketBatchOptions {
     replicationLock: MongoSyncRulesLock;
     opIdAllocator: MongoOpIdAllocator;
-    db: VersionedPowerSyncMongo;
     /**
      * The parsed sync config set for this batch.
      *
@@ -53,8 +54,8 @@ export interface MongoBucketBatchOptions {
 export declare abstract class MongoBucketBatch extends BaseObserver<storage.BucketBatchStorageListener> implements storage.BucketStorageBatch {
     protected readonly options: MongoBucketBatchOptions;
     protected logger: Logger;
-    private readonly client;
     readonly db: VersionedPowerSyncMongo;
+    /** Shared by this lease's writers; only use inside a serialized public operation. */
     readonly session: mongo.ClientSession;
     protected readonly sync_rules: HydratedSyncConfig;
     protected readonly replicationStreamId: number;
@@ -68,15 +69,15 @@ export declare abstract class MongoBucketBatch extends BaseObserver<storage.Buck
      * via SourceTable.storeCurrentData. The effective per-record value is the conjunction of
      * the two.
      */
-    private readonly storeCurrentData;
+    protected readonly storeCurrentData: boolean;
     readonly skipExistingRows: boolean;
     protected readonly mapping: BucketDefinitionMapping;
     protected readonly objectStorageUsageWriterId: string;
-    private batch;
+    protected batch: OperationBatch | null;
     protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[];
-    private markRecordUnavailable;
-    private hooks;
-    private clearedError;
+    protected readonly markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
+    protected hooks: storage.StorageHooks | undefined;
+    protected clearedError: boolean;
     protected tracer: PerformanceTracer<'storage' | 'evaluate'>;
     /**
      * Last written op, if any. This may not reflect a consistent checkpoint.
@@ -95,56 +96,73 @@ export declare abstract class MongoBucketBatch extends BaseObserver<storage.Buck
     resumeFromLsn: string | null;
     constructor(options: MongoBucketBatchOptions);
     addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void;
-    abstract resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
+    resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
+    protected abstract resolveTablesInternal(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
     protected abstract createPersistedBatch(writtenSize: number): PersistedBatch;
     protected abstract get sourceRecordStore(): SourceRecordStore;
     protected abstract cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]): Promise<void>;
-    abstract commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
-    abstract keepalive(lsn: string): Promise<storage.CheckpointResult>;
-    abstract setResumeLsn(lsn: string): Promise<void>;
-    abstract getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
-    abstract markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
-    abstract markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
+    commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
+    protected abstract commitInternal(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
+    keepalive(lsn: string): Promise<storage.CheckpointResult>;
+    setResumeLsn(lsn: string): Promise<void>;
+    protected abstract setResumeLsnInternal(lsn: string): Promise<void>;
+    getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
+    protected abstract getSourceTableStatusInternal(table: storage.SourceTable): Promise<storage.SourceTable | null>;
+    markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
+    protected abstract markAllSnapshotDoneInternal(no_checkpoint_before_lsn: string): Promise<void>;
+    markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
         throwOnConflict?: boolean;
     }): Promise<void>;
-    abstract markTableSnapshotRequired(table: storage.SourceTable): Promise<void>;
-    abstract markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
-    protected abstract batchCreateCustomWriteCheckpoints(session: mongo.ClientSession, opId: InternalOpId): Promise<void>;
+    protected abstract markSnapshotDoneInternal(no_checkpoint_before_lsn: string, options?: {
+        throwOnConflict?: boolean;
+    }): Promise<void>;
+    markTableSnapshotRequired(table: storage.SourceTable): Promise<void>;
+    protected abstract markTableSnapshotRequiredInternal(table: storage.SourceTable): Promise<void>;
+    markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
+    protected abstract markTableSnapshotDoneInternal(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
     /** Perform any version-specific setup required before writing custom checkpoints. */
     protected prepareCustomWriteCheckpoints(): Promise<void>;
     flush(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null>;
+    /** Internal helpers require the public operation to hold the writer mutex. */
+    protected flushInternal(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null>;
     /** Publish the checkpoint in the final flush transaction, or use the empty-commit path. */
-    protected flushAndCommit<T>(checkpoint: (stream: SyncRuleDocumentBase, lastOp: InternalOpId) => Promise<T>, commitWithoutFlush: () => Promise<T>, options?: storage.BatchBucketFlushOptions, projection?: mongo.Document): Promise<T>;
-    private flushInner;
-    private replicateBatch;
-    private saveOperation;
+    protected flushAndCommit<T>(checkpoint: (stream: SyncRuleDocumentBase, lastOp: InternalOpId, writes?: MongoWriteBatch) => Promise<T>, commitWithoutFlush: () => Promise<T>, options?: storage.BatchBucketFlushOptions, projection?: mongo.Document): Promise<T>;
+    protected abstract flushInner(options?: storage.BatchBucketFlushOptions, checkpoint?: (stream: SyncRuleDocumentBase, lastOp: InternalOpId, writes?: MongoWriteBatch) => Promise<void>, projection?: mongo.Document): Promise<storage.FlushedResult | null>;
+    /**
+     * Evaluate one source operation and append its storage writes to batch.
+     *
+     * Diagnostics run during evaluation. V1 calls markRecordUnavailable immediately;
+     * v3/v4 supply a collector and deliver repair notifications only after a
+     * confirmed publication. A failed row attempt may have registered a repair,
+     * so its caller must discard that row's collected notification on failure.
+     */
+    protected saveOperation(batch: PersistedBatch, operation: RecordOperation, sourceRecord: LoadedSourceRecord | null, opSeq: MongoIdSequence, onRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined): LoadedSourceRecord | null;
     protected fence(session?: mongo.ClientSession, projection?: mongo.Document): Promise<mongo.WithId<SyncRuleDocumentBase>>;
     protected withTransaction<T>(cb: (stream: SyncRuleDocumentBase) => Promise<T>, session?: mongo.ClientSession): Promise<T>;
-    /** The first operation must modify the stream document conditional on lease ownership. */
-    protected withFencedTransaction<S, T>(acquireFence: () => Promise<S>, cb: (stream: S) => Promise<T>, session?: mongo.ClientSession): Promise<T>;
+    /**
+     * Internal transaction helper: the public writer operation already holds the
+     * mutex. The first operation must modify the stream document conditional on
+     * lease ownership. This database fence is required on every retry even though
+     * local writers are serialized; another process may have taken over the lease.
+     */
+    protected withFencedTransaction<S, T>(acquireFence: () => Promise<S>, cb: (stream: S) => Promise<T>, session?: mongo.ClientSession, description?: string): Promise<T>;
+    /**
+     * Public database operations acquire the lease's mutex once, then call only
+     * internal helpers until they finish. Those helpers must never call a public
+     * database operation or reacquire this mutex. All session use, including reads,
+     * belongs inside this boundary so it cannot join another writer's transaction.
+     * Transaction helpers still fence database ownership on every attempt.
+     */
     private withWriterLock;
-    /** Single-document updates enforce ownership atomically, without a separate transaction. */
+    /** Internal single-document update; the public operation already holds the mutex. */
     protected updateStreamMetadata(set: mongo.Document, filter?: mongo.Document, writeConcern?: mongo.WriteConcernSettings): Promise<void>;
     /** Caller must already be inside a fenced transaction holding the writer mutex. */
     protected updateStreamMetadataInTransaction(set: mongo.Document, filter?: mongo.Document): Promise<void>;
     private writeStreamMetadata;
-    private withReplicationTransaction;
     /** Highest durably persisted operation, including operations not yet checkpointed. */
     protected abstract persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId;
     /** Advance the persisted head in the same transaction as the operations. */
     protected abstract onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void;
-    /**
-     * Called after a replication transaction has successfully committed, with the last persisted op id.
-     *
-     * v1 storage tracks this in memory to fold into the next checkpoint. v3 storage does not need it:
-     * the stream-level `last_persisted_op` is already advanced durably by
-     * {@link onReplicationTransactionFlush} or the combined checkpoint update within the same
-     * transaction, and empty commits read it from the document.
-     *
-     * Keep calls to this adjacent to {@link withReplicationTransaction} usage - both must observe
-     * every path that persists ops.
-     */
-    protected recordPersistedOp(_lastOp: InternalOpId): void;
     [Symbol.asyncDispose](): Promise<void>;
     dispose(): Promise<void>;
     save(record: storage.SaveOptions): Promise<storage.FlushedResult | null>;
@@ -152,10 +170,13 @@ export declare abstract class MongoBucketBatch extends BaseObserver<storage.Buck
      * Drop is equivalent to TRUNCATE, plus removing our record of the table.
      */
     drop(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
+    protected dropInternal(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
     truncate(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
+    protected truncateInternal(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
     truncateSingle(sourceTable: storage.SourceTable): Promise<InternalOpId>;
+    protected abstract truncateSingleInternal(sourceTable: storage.SourceTable): Promise<InternalOpId>;
     updateTableProgress(table: storage.SourceTable, progress: Partial<storage.TableSnapshotStatus>): Promise<storage.SourceTable>;
-    protected clearError(session: mongo.ClientSession): Promise<void>;
+    private updateTableProgressInternal;
     /**
      * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable}
      */