@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

package/dist/storage/implementation/MongoIdSequence.js

@@ -6,18 +6,31 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework';
  * at the end of the transaction.
  */
 export class MongoIdSequence {
+    ranges;
     _last;
-    constructor(last) {
+    constructor(last, ranges) {
+        this.ranges = ranges;
         if (typeof last != 'bigint') {
             throw new ReplicationAssertionError(`BigInt required, got ${last} ${typeof last}`);
         }
         this._last = last;
     }
     next() {
+        if (this.ranges != null) {
+            const next = this.ranges.find((range) => range.end > this._last);
+            if (next == null) {
+                throw new OpIdRangeExhausted();
+            }
+            this._last = this._last + 1n < next.start ? next.start : this._last + 1n;
+            return this._last;
+        }
         return ++this._last;
     }
     last() {
         return this._last;
     }
 }
+/** Abort the transaction before reserving more IDs and retrying its evaluation. */
+export class OpIdRangeExhausted extends Error {
+}
 //# sourceMappingURL=MongoIdSequence.js.map

package/dist/storage/implementation/MongoIdSequence.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoIdSequence.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoIdSequence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IAClB,KAAK,CAAS;IAEtB,YAAY,IAAY;QACtB,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,yBAAyB,CAAC,wBAAwB,IAAI,IAAI,OAAO,IAAI,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAI;QACF,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF"}
+{"version":3,"file":"MongoIdSequence.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoIdSequence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IAKP;IAJX,KAAK,CAAS;IAEtB,YACE,IAAY,EACK,MAA6B;QAA7B,WAAM,GAAN,MAAM,CAAuB;QAE9C,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,yBAAyB,CAAC,wBAAwB,IAAI,IAAI,OAAO,IAAI,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACjE,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,kBAAkB,EAAE,CAAC;YACjC,CAAC;YACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YACzE,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QACD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF;AAQD,mFAAmF;AACnF,MAAM,OAAO,kBAAmB,SAAQ,KAAK;CAAG"}

package/dist/storage/implementation/MongoOpIdAllocator.d.ts

@@ -0,0 +1,27 @@
+import { VersionedPowerSyncMongo } from './db.js';
+import { MongoIdSequence } from './MongoIdSequence.js';
+/**
+ * Shared by writers of one stream and lease owner in this process. Reservations
+ * use short majority writes, never the publication session. A crashed owner
+ * abandons its unused ranges; the global sequence is an allocation watermark.
+ */
+export declare class MongoOpIdAllocator {
+    private readonly db;
+    private ranges;
+    private reserving?;
+    constructor(db: VersionedPowerSyncMongo);
+    discard(): void;
+    sequence(persistedOp: bigint): MongoIdSequence;
+    committed(lastOp: bigint): void;
+    /** Top up between transactions, retaining every unused ID in existing ranges. */
+    ensureCapacity(): Promise<void>;
+    reserve(): Promise<void>;
+    /**
+     * Atomically reserve a range of operation IDs in the global sequence.
+     *
+     * We continue using the range across multiple batches, until the range is exhausted,
+     * so we don't "waste" large segements in long-running processes with small batches.
+     * However, we can't return unused portions back to the database if we don't use them.
+     */
+    private reserveRange;
+}

package/dist/storage/implementation/MongoOpIdAllocator.js

@@ -0,0 +1,97 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { MongoIdSequence } from './MongoIdSequence.js';
+const RESERVATION_SIZE = 65536n;
+const LOW_WATER_MARK = 16384n;
+const MAX_OP_ID = (1n << 63n) - 1n;
+/**
+ * Shared by writers of one stream and lease owner in this process. Reservations
+ * use short majority writes, never the publication session. A crashed owner
+ * abandons its unused ranges; the global sequence is an allocation watermark.
+ */
+export class MongoOpIdAllocator {
+    db;
+    ranges = [];
+    reserving;
+    constructor(db) {
+        this.db = db;
+    }
+    discard() {
+        this.ranges = [];
+    }
+    sequence(persistedOp) {
+        // The stream is fenced before reading this head. Another writer may have
+        // committed since this allocator was last used, including in another process.
+        // Never reuse reserved IDs below that durable head.
+        return new MongoIdSequence(persistedOp, this.ranges.slice());
+    }
+    committed(lastOp) {
+        this.ranges = this.ranges
+            .filter((range) => range.end > lastOp)
+            .map((range) => ({ start: range.start > lastOp ? range.start : lastOp + 1n, end: range.end }));
+    }
+    /** Top up between transactions, retaining every unused ID in existing ranges. */
+    async ensureCapacity() {
+        const remaining = this.ranges.reduce((total, range) => total + range.end - range.start + 1n, 0n);
+        if (remaining < LOW_WATER_MARK) {
+            await this.reserve();
+        }
+    }
+    async reserve() {
+        if (this.reserving != null) {
+            return this.reserving;
+        }
+        this.reserving = this.reserveRange();
+        try {
+            await this.reserving;
+        }
+        finally {
+            this.reserving = undefined;
+        }
+    }
+    /**
+     * Atomically reserve a range of operation IDs in the global sequence.
+     *
+     * We continue using the range across multiple batches, until the range is exhausted,
+     * so we don't "waste" large segements in long-running processes with small batches.
+     * However, we can't return unused portions back to the database if we don't use them.
+     */
+    async reserveRange() {
+        try {
+            // Return the previous watermark so an unchanged value at the limit can be
+            // distinguished from a successful reservation ending near the limit.
+            const previous = await this.db.op_id_sequence.findOneAndUpdate({ _id: 'main' }, [
+                {
+                    $set: {
+                        op_id: {
+                            $let: {
+                                vars: { current: { $ifNull: ['$op_id', 0n] } },
+                                in: {
+                                    $cond: [
+                                        { $lte: ['$$current', MAX_OP_ID - RESERVATION_SIZE] },
+                                        { $add: ['$$current', RESERVATION_SIZE] },
+                                        '$$current'
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                }
+            ], { upsert: true, returnDocument: 'before', writeConcern: { w: 'majority' } });
+            const last = previous?.op_id ?? 0n;
+            if (last > MAX_OP_ID - RESERVATION_SIZE) {
+                // Not expected to ever happen, unless the sequence was manually modified
+                throw new ReplicationAssertionError('Operation ID sequence exhausted');
+            }
+            this.ranges.push({ start: last + 1n, end: last + RESERVATION_SIZE });
+        }
+        catch (error) {
+            // Concurrent first reservations may race to insert the sequence document - retry in that case.
+            if (error instanceof mongo.MongoServerError && error.code === 11000) {
+                return this.reserveRange();
+            }
+            throw error;
+        }
+    }
+}
+//# sourceMappingURL=MongoOpIdAllocator.js.map

package/dist/storage/implementation/MongoOpIdAllocator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoOpIdAllocator.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoOpIdAllocator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E,OAAO,EAAE,eAAe,EAAa,MAAM,sBAAsB,CAAC;AAElE,MAAM,gBAAgB,GAAG,MAAO,CAAC;AACjC,MAAM,cAAc,GAAG,MAAO,CAAC;AAC/B,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;AAEnC;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAIA;IAHrB,MAAM,GAAgB,EAAE,CAAC;IACzB,SAAS,CAAiB;IAElC,YAA6B,EAA2B;QAA3B,OAAE,GAAF,EAAE,CAAyB;IAAG,CAAC;IAE5D,OAAO;QACL,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,CAAC;IAED,QAAQ,CAAC,WAAmB;QAC1B,yEAAyE;QACzE,8EAA8E;QAC9E,oDAAoD;QACpD,OAAO,IAAI,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,SAAS,CAAC,MAAc;QACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;aACtB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC;aACrC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACnG,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,cAAc;QAClB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QACjG,IAAI,SAAS,GAAG,cAAc,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,SAAS,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,SAAS,CAAC;QACvB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC;YACH,0EAA0E;YAC1E,qEAAqE;YACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC5D,EAAE,GAAG,EAAE,MAAM,EAAE,EACf;gBACE;oBACE,IAAI,EAAE;wBACJ,KAAK,EAAE;4BACL,IAAI,EAAE;gCACJ,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE;gCAC9C,EAAE,EAAE;oCACF,KAAK,EAAE;wCACL,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,SAAS,GAAG,gBAAgB,CAAC,EAAE;wCACrD,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE;wCACzC,WAAW;qCACZ;iCACF;6BACF;yBACF;qBACF;iBACF;aACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,CAC5E,CAAC;YACF,MAAM,IAAI,GAAG,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,IAAI,GAAG,SAAS,GAAG,gBAAgB,EAAE,CAAC;gBACxC,yEAAyE;gBACzE,MAAM,IAAI,yBAAyB,CAAC,iCAAiC,CAAC,CAAC;YACzE,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,GAAG,gBAAgB,EAAE,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,+FAA+F;YAC/F,IAAI,KAAK,YAAY,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACpE,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}

package/dist/storage/implementation/MongoParameterCompactor.d.ts

@@ -1,23 +1,96 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { CompactOptions, InternalOpId } from '@powersync/service-core';
+import { Logger } from '@powersync/lib-services-framework';
+import { bson, CompactOptions, InternalOpId } from '@powersync/service-core';
 import type { VersionedPowerSyncMongo } from './db.js';
+type ParameterCompactionReadDocument = {
+    _id: InternalOpId;
+    key: mongo.Document;
+    lookup: bson.Binary;
+    bucket_parameters?: unknown[] | null;
+};
+export type ParameterCompactionResult = {
+    collections: number;
+    scannedEntries: number;
+    deletedEntries: number;
+};
 /**
  * Compacts parameter lookup data (the bucket_parameters collection).
  *
- * This scans through the entire collection to find data to compact.
+ * Both storage versions persist a per-stream compaction cursor, so a run only scans entries in the
+ * un-compacted operation-id range. V1 scans its shared collection using only the `_id` index, so it
+ * additionally filters by stream in code.
+ *
+ * The cursor is a single value covering every collection of the stream, so it can only be advanced
+ * to a boundary that all collections have passed. To keep it moving during a long pass, collections
+ * are processed in lock-step rather than one after another - see {@link compactCollections}.
  *
  * For background, see the `/docs/storage/parameter-lookups.md` file.
  */
-export declare class MongoParameterCompactor {
+export declare abstract class MongoParameterCompactor {
+    #private;
     protected readonly db: VersionedPowerSyncMongo;
-    protected readonly group_id: number;
+    protected readonly replicationStreamId: number;
     protected readonly checkpoint: InternalOpId;
     protected readonly options: CompactOptions;
-    protected readonly getCollectionsCb?: (() => Promise<mongo.Collection<mongo.Document>[]>) | undefined;
-    constructor(db: VersionedPowerSyncMongo, group_id: number, checkpoint: InternalOpId, options: CompactOptions, getCollectionsCb?: (() => Promise<mongo.Collection<mongo.Document>[]>) | undefined);
+    protected readonly parameterCompactionBatchSize: number;
+    protected readonly parameterCompactionPersistIntervalMs: number;
+    protected readonly logger: Logger;
+    protected readonly signal?: AbortSignal;
+    constructor(db: VersionedPowerSyncMongo, replicationStreamId: number, checkpoint: InternalOpId, options: CompactOptions, parameterCompactionBatchSize?: number, parameterCompactionPersistIntervalMs?: number);
     compact(): Promise<void>;
-    protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
-    protected collectionFilter(): mongo.Document;
-    protected deleteFilter(doc: mongo.Document): mongo.Document;
-    protected compactCollection(collection: mongo.Collection<mongo.Document>): Promise<void>;
+    /**
+     * The exclusive operation-id boundary through which this stream's parameter indexes have all
+     * been compacted.
+     */
+    protected readCompactedBefore(): Promise<InternalOpId>;
+    protected persistCompactedBefore(compactedBefore: InternalOpId): Promise<void>;
+    /**
+     * Commits the checkpoint-change invalidation fence before the first delete of this pass.
+     *
+     * Checkpoint change detection finds changed lookups by querying parameter entries in
+     * (lastCheckpoint, nextCheckpoint]. Compaction physically removes entries in that range, so a
+     * checkpoint that can no longer see the full history must instead invalidate all parameter
+     * buckets. The fence records the boundary below which that history may be missing.
+     *
+     * The fence must be committed before the first delete: MongoDB snapshot ordering then
+     * guarantees that a checkpoint snapshot which observes a deletion also observes the fence.
+     *
+     * This deliberately isn't the same value as the compaction cursor. If the pass fails halfway,
+     * the fence only causes conservative invalidation, while an advanced cursor would skip
+     * deletion work that never completed.
+     */
+    private ensureInvalidationFence;
+    protected abstract getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+    protected abstract shouldCompactDocument(doc: ParameterCompactionReadDocument): boolean;
+    /** Deletes history preceding a batch for several identities sharing a lookup. */
+    protected abstract leadingHistoryDeleteFilter(lookup: bson.Binary, keys: mongo.Document[], before: InternalOpId): mongo.Document;
+    /**
+     * Processes every collection of the stream, in lock-step: each turn takes one batch from the
+     * collection that has processed the least so far.
+     *
+     * The persisted cursor is the minimum position over all collections, which is exactly the
+     * boundary that all of them have passed, so it can be advanced periodically during the pass.
+     * Always picking the collection that is furthest behind also keeps each of them within one batch
+     * of that boundary, bounding the work an interrupted pass has to repeat.
+     *
+     * V1 storage always has a single collection; V3 has a collection per defined index. So in V1 the
+     * ame process collapses to compacting the single collection in order, while V3 can alternate between
+     * collections.
+     */
+    private compactCollections;
+    /**
+     * The boundary that every scope has processed past, capped at the target checkpoint, and the
+     * scope sitting on it.
+     *
+     * The frontier is the furthest the cursor may be advanced. The scope is the one that has
+     * processed the least, or null once all of them have reached the target checkpoint.
+     */
+    private frontier;
+    /**
+     * Reads and processes one batch from the scope, and advances its position past that batch.
+     */
+    private compactBatch;
+    /** Deletes documents by `_id`, chunked to bound the command size. Returns the number deleted. */
+    private deleteByIds;
 }
+export {};