@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

package/dist/storage/implementation/CheckpointState.d.ts

@@ -0,0 +1,20 @@
+export interface CheckpointStateInput {
+    lsn: string;
+    snapshotDone: boolean;
+    lastCheckpointLsn: string | null;
+    noCheckpointBefore: string | null;
+    keepaliveOp: bigint | null;
+    lastCheckpoint: bigint | null;
+    persistedOp: bigint | null;
+    createEmptyCheckpoints: boolean;
+}
+export interface CheckpointStateResult {
+    canCheckpoint: boolean;
+    checkpointBlocked: boolean;
+    checkpointCreated: boolean;
+    notEmpty: boolean;
+    newKeepaliveOp: bigint | null;
+    newLastCheckpoint: bigint | null;
+}
+export declare function canCheckpointState(lsn: string, state: Pick<CheckpointStateInput, 'snapshotDone' | 'lastCheckpointLsn' | 'noCheckpointBefore'>): boolean;
+export declare function calculateCheckpointState(input: CheckpointStateInput): CheckpointStateResult;

package/dist/storage/implementation/CheckpointState.js

@@ -0,0 +1,31 @@
+function maxOpId(...values) {
+    let max = 0n;
+    for (const value of values) {
+        if (value != null && value > max) {
+            max = value;
+        }
+    }
+    return max;
+}
+export function canCheckpointState(lsn, state) {
+    return (state.snapshotDone === true &&
+        (state.lastCheckpointLsn == null || state.lastCheckpointLsn <= lsn) &&
+        (state.noCheckpointBefore == null || state.noCheckpointBefore <= lsn));
+}
+export function calculateCheckpointState(input) {
+    const canCheckpoint = canCheckpointState(input.lsn, input);
+    const newKeepaliveOp = canCheckpoint ? null : maxOpId(input.keepaliveOp, input.persistedOp);
+    const newLastCheckpoint = canCheckpoint
+        ? maxOpId(input.lastCheckpoint, input.persistedOp, input.keepaliveOp)
+        : input.lastCheckpoint;
+    const notEmpty = input.createEmptyCheckpoints || input.keepaliveOp !== newKeepaliveOp || input.lastCheckpoint !== newLastCheckpoint;
+    return {
+        canCheckpoint,
+        checkpointBlocked: !canCheckpoint,
+        checkpointCreated: canCheckpoint && notEmpty,
+        notEmpty,
+        newKeepaliveOp,
+        newLastCheckpoint
+    };
+}
+//# sourceMappingURL=CheckpointState.js.map

package/dist/storage/implementation/CheckpointState.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CheckpointState.js","sourceRoot":"","sources":["../../../src/storage/implementation/CheckpointState.ts"],"names":[],"mappings":"AAoBA,SAAS,OAAO,CAAC,GAAG,MAAqC;IACvD,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;YACjC,GAAG,GAAG,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,GAAW,EACX,KAA8F;IAE9F,OAAO,CACL,KAAK,CAAC,YAAY,KAAK,IAAI;QAC3B,CAAC,KAAK,CAAC,iBAAiB,IAAI,IAAI,IAAI,KAAK,CAAC,iBAAiB,IAAI,GAAG,CAAC;QACnE,CAAC,KAAK,CAAC,kBAAkB,IAAI,IAAI,IAAI,KAAK,CAAC,kBAAkB,IAAI,GAAG,CAAC,CACtE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,KAA2B;IAClE,MAAM,aAAa,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3D,MAAM,cAAc,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5F,MAAM,iBAAiB,GAAG,aAAa;QACrC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC;QACrE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC;IACzB,MAAM,QAAQ,GACZ,KAAK,CAAC,sBAAsB,IAAI,KAAK,CAAC,WAAW,KAAK,cAAc,IAAI,KAAK,CAAC,cAAc,KAAK,iBAAiB,CAAC;IAErH,OAAO;QACL,aAAa;QACb,iBAAiB,EAAE,CAAC,aAAa;QACjC,iBAAiB,EAAE,aAAa,IAAI,QAAQ;QAC5C,QAAQ;QACR,cAAc;QACd,iBAAiB;KAClB,CAAC;AACJ,CAAC"}

package/dist/storage/implementation/MongoBucketBatch.d.ts

@@ -1,44 +1,59 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { HydratedSyncRules, SqlEventDescriptor } from '@powersync/service-sync-rules';
+import { HydratedSyncConfig, SqlEventDescriptor } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import { BaseObserver, Logger } from '@powersync/lib-services-framework';
-import { BucketStorageMarkRecordUnavailable, CheckpointResult, InternalOpId, storage } from '@powersync/service-core';
-import { VersionedPowerSyncMongo } from './db.js';
-import { CurrentBucket } from './models.js';
-/**
- * 15MB
- */
-export declare const MAX_ROW_SIZE: number;
-export declare const EMPTY_DATA: mongo.Binary;
+import { BucketStorageMarkRecordUnavailable, InternalOpId, PerformanceTracer, storage } from '@powersync/service-core';
+import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
+import { PersistedBatch } from './common/PersistedBatch.js';
+import { SourceRecordStore } from './common/SourceRecordStore.js';
+import type { VersionedPowerSyncMongo } from './db.js';
 export interface MongoBucketBatchOptions {
     db: VersionedPowerSyncMongo;
-    syncRules: HydratedSyncRules;
+    syncRules: HydratedSyncConfig;
     groupId: number;
     slotName: string;
+    syncConfigId?: bson.ObjectId | null;
     lastCheckpointLsn: string | null;
     keepaliveOp: InternalOpId | null;
     resumeFromLsn: string | null;
     storeCurrentData: boolean;
+    mapping: BucketDefinitionMapping;
     /**
      * Set to true for initial replication.
      */
     skipExistingRows: boolean;
     markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
-    logger?: Logger;
+    hooks: storage.StorageHooks | undefined;
+    logger: Logger;
+    tracer?: PerformanceTracer<'storage' | 'evaluate'>;
 }
-export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchStorageListener> implements storage.BucketStorageBatch {
-    private logger;
+export declare abstract class MongoBucketBatch extends BaseObserver<storage.BucketBatchStorageListener> implements storage.BucketStorageBatch {
+    protected readonly options: MongoBucketBatchOptions;
+    protected logger: Logger;
     private readonly client;
     readonly db: VersionedPowerSyncMongo;
     readonly session: mongo.ClientSession;
-    private readonly sync_rules;
-    private readonly group_id;
+    protected readonly sync_rules: HydratedSyncConfig;
+    protected readonly group_id: number;
     private readonly slot_name;
+    /**
+     * Source-level setting for whether raw row data should be stored in current_data.
+     *
+     * Some sources always send complete rows (MongoDB, MySQL with binlog_row_image=full),
+     * in which case this is false for the whole batch. For sources where it depends on the
+     * table (Postgres REPLICA IDENTITY), this is true and the decision is refined per-table
+     * via SourceTable.storeCurrentData. The effective per-record value is the conjunction of
+     * the two.
+     */
     private readonly storeCurrentData;
-    private readonly skipExistingRows;
+    readonly skipExistingRows: boolean;
+    protected readonly mapping: BucketDefinitionMapping;
     private batch;
     private write_checkpoint_batch;
     private markRecordUnavailable;
+    private hooks;
     private clearedError;
+    private tracer;
     /**
      * Last LSN received associated with a checkpoint.
      *
@@ -46,8 +61,8 @@ export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchSt
      * 1. A commit LSN.
      * 2. A keepalive message LSN.
      */
-    private last_checkpoint_lsn;
-    private persisted_op;
+    protected last_checkpoint_lsn: string | null;
+    protected persisted_op: InternalOpId | null;
     /**
      * Last written op, if any. This may not reflect a consistent checkpoint.
      */
@@ -63,29 +78,31 @@ export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchSt
      * This is set when creating the batch, but may not be updated afterwards.
      */
     resumeFromLsn: string | null;
-    private needsActivation;
     constructor(options: MongoBucketBatchOptions);
     addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void;
     get lastCheckpointLsn(): string | null;
+    abstract resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
+    protected abstract createPersistedBatch(writtenSize: number): PersistedBatch;
+    protected abstract get sourceRecordStore(): SourceRecordStore;
+    protected abstract cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]): Promise<void>;
+    abstract commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
+    abstract keepalive(lsn: string): Promise<storage.CheckpointResult>;
+    abstract setResumeLsn(lsn: string): Promise<void>;
+    abstract getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
+    abstract markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
+    abstract markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
+        throwOnConflict?: boolean;
+    }): Promise<void>;
+    abstract markTableSnapshotRequired(table: storage.SourceTable): Promise<void>;
+    abstract markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
     flush(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null>;
     private flushInner;
     private replicateBatch;
     private saveOperation;
-    private withTransaction;
+    protected withTransaction(cb: () => Promise<void>): Promise<void>;
     private withReplicationTransaction;
     [Symbol.asyncDispose](): Promise<void>;
     dispose(): Promise<void>;
-    private lastWaitingLogThottled;
-    commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<CheckpointResult>;
-    private cleanupCurrentData;
-    /**
-     * Switch from processing -> active if relevant.
-     *
-     * Called on new commits.
-     */
-    private autoActivate;
-    keepalive(lsn: string): Promise<CheckpointResult>;
-    setResumeLsn(lsn: string): Promise<void>;
     save(record: storage.SaveOptions): Promise<storage.FlushedResult | null>;
     /**
      * Drop is equivalent to TRUNCATE, plus removing our record of the table.
@@ -94,13 +111,9 @@ export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchSt
     truncate(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
     truncateSingle(sourceTable: storage.SourceTable): Promise<InternalOpId>;
     updateTableProgress(table: storage.SourceTable, progress: Partial<storage.TableSnapshotStatus>): Promise<storage.SourceTable>;
-    markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
-    markTableSnapshotRequired(table: storage.SourceTable): Promise<void>;
-    markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
     protected clearError(): Promise<void>;
     /**
      * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
      */
     protected getTableEvents(table: storage.SourceTable): SqlEventDescriptor[];
 }
-export declare function currentBucketKey(b: CurrentBucket): string;