@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

package/dist/storage/implementation/v3/compact-utils.js

@@ -0,0 +1,223 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { addChecksums } from '@powersync/service-core';
+import { AVAILABLE_LEASE_EXPR } from './CompactionLease.js';
+/**
+ * Perform chunk compaction if at least this many chunks have been added since
+ * the latest compaction.
+ *
+ * A lower value increases compaction frequency and can increase chunk rewrites
+ * and object-storage operations. A higher value leaves more small chunks for
+ * longer, which can hurt sync performance.
+ */
+const MERGE_CHUNKS_THRESHOLD = 8;
+export const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000;
+export var CompactionKind;
+(function (CompactionKind) {
+    CompactionKind["Full"] = "full";
+    CompactionKind["Chunks"] = "chunks";
+})(CompactionKind || (CompactionKind = {}));
+export class CompactionContext {
+    lease;
+    kind;
+    decision;
+    rescheduleNotBefore;
+    targetOp;
+    constructor(lease, kind, decision, rescheduleNotBefore, targetOp) {
+        this.lease = lease;
+        this.kind = kind;
+        this.decision = decision;
+        this.rescheduleNotBefore = rescheduleNotBefore;
+        this.targetOp = targetOp;
+    }
+    get state() {
+        return this.lease.state;
+    }
+    get startedAt() {
+        return this.lease.startedAt;
+    }
+    get lastOp() {
+        return this.lease.lastOp;
+    }
+}
+export function bucketStats(state) {
+    return {
+        count: state.bucket_stats.count,
+        bytes: state.bucket_stats.bytes,
+        chunks: state.bucket_stats.chunks
+    };
+}
+export function emptyBucketStats() {
+    return { count: 0, bytes: 0n, chunks: 0, checksum: 0 };
+}
+export function statsForDocument(document) {
+    return {
+        count: document.count,
+        bytes: BigInt(document.size),
+        chunks: 1,
+        checksum: addChecksums(0, Number(document.checksum))
+    };
+}
+export function statsForDocuments(documents) {
+    let result = emptyBucketStats();
+    for (const document of documents) {
+        result = combineAdjacentStats(result, statsForDocument(document));
+    }
+    return result;
+}
+/** A scheduled bucket always has writes awaiting a full compact. */
+export function firstUncompactedWrite(state) {
+    if (state.first_uncompacted_write == null) {
+        throw new ReplicationAssertionError(`Scheduled V3 bucket ${state._id.b} has no first uncompacted write`);
+    }
+    return state.first_uncompacted_write;
+}
+export function combineChunkStats(previous, compactedTail, overlappingCompactedChunk) {
+    return {
+        count: previous.count - overlappingCompactedChunk.count + compactedTail.count,
+        bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes,
+        chunks: previous.chunks - 1 + compactedTail.chunks,
+        checksum: addChecksums(addChecksums(Number(previous.checksum), -overlappingCompactedChunk.checksum), compactedTail.checksum)
+    };
+}
+export function combineAdjacentStats(first, second) {
+    return {
+        count: first.count + second.count,
+        bytes: first.bytes + second.bytes,
+        chunks: first.chunks + second.chunks,
+        checksum: addChecksums(first.checksum, second.checksum)
+    };
+}
+export function applyCompactionDelta(total, before, after) {
+    return {
+        count: total.count - before.count + after.count,
+        bytes: total.bytes - before.bytes + after.bytes,
+        chunks: total.chunks - before.chunks + after.chunks
+    };
+}
+/** Replace one stored range in an accumulated statistics snapshot. */
+export function applyStatsReplacement(total, before, after) {
+    return {
+        ...applyCompactionDelta(total, before, after),
+        checksum: addChecksums(addChecksums(total.checksum, -before.checksum), after.checksum)
+    };
+}
+/**
+ * This checks that the bucket state hasn't changed by a concurrent write since we checked.
+ *
+ * If it has changed, we'll re-check in the next batch.
+ */
+export function unclaimedSnapshotFilter(state) {
+    return {
+        _id: state._id,
+        last_op: state.last_op,
+        next_compact_check: state.next_compact_check,
+        first_uncompacted_write: state.first_uncompacted_write ?? { $exists: false },
+        bucket_stats: state.bucket_stats,
+        compacted_state: state.compacted_state ?? { $exists: false },
+        last_full_compact: state.last_full_compact ?? { $exists: false },
+        ...AVAILABLE_LEASE_EXPR
+    };
+}
+/**
+ * Calculate the earliest full compaction time from the first uncompacted
+ * write, bounded by the maximum retention interval.
+ */
+export function fullCompactionCheckAt(state, config) {
+    const firstWrite = firstUncompactedWrite(state);
+    const stats = bucketStats(state);
+    const lastFull = state.last_full_compact;
+    // The number of operations since the last full compact.
+    // We may make this more specific in the future, to track new updates and deletes only, ignoring
+    // full new inserts, but that requires more granular tracking when replicating.
+    const uncompactedCount = lastFull == null ? stats.count : Math.max(0, stats.count - lastFull.count);
+    const compactedRows = lastFull?.puts ?? 0;
+    // If no full compact has ever been performed: ratio = 1, compact after minCompactFullIntervalMs.
+    // If every row has been updated or deleted exactly once since the last full compact: ratio = 0.5, compact after minCompactFullIntervalMs * 2.
+    // If 10% of rows has been updated since the last full compact, compact after minCompactFullIntervalMs * 11.
+    // If every row has been updated multiple times, the ratio tends closer to 1 again.
+    const ratio = uncompactedCount == 0 ? 0 : uncompactedCount / (compactedRows + uncompactedCount);
+    const fullIntervalMs = ratio == 0 ? config.maxCompactFullIntervalMs : config.minCompactFullIntervalMs / ratio;
+    return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, config.maxCompactFullIntervalMs));
+}
+export function chooseCompactionKind(state, now, config) {
+    // For chunk compaction, we consider the number of chunks added.
+    // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added.
+    // A future policy could also use bytes per chunk or records per chunk to
+    // decide when to compact.
+    const fullCheckAt = fullCompactionCheckAt(state, config);
+    // Schedule a little late so a worker using a slightly earlier clock does
+    // not wake before the exact full-compaction condition is true.
+    const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS);
+    const compacted = state.compacted_state;
+    const shouldCompactChunks = hasEnoughChunksToMerge(state);
+    const canCheckChunks = compacted == null || now.getTime() - compacted.at.getTime() >= config.minCompactChunkIntervalMs;
+    // Too few new chunks cannot make chunk compaction eligible. Do not poll this
+    // bucket at the chunk-compaction interval; only wake it for its full-compact check.
+    const nextCompactCheck = !shouldCompactChunks
+        ? fullCheckWithMargin
+        : {
+            $min: [
+                fullCheckWithMargin,
+                { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: config.minCompactChunkIntervalMs } }
+            ]
+        };
+    let kind = null;
+    if (now >= fullCheckAt) {
+        kind = CompactionKind.Full;
+    }
+    else if (canCheckChunks && shouldCompactChunks) {
+        kind = CompactionKind.Chunks;
+    }
+    return {
+        kind,
+        nextCompactCheck
+    };
+}
+export function chooseRequestedCompactionKind(state, requestedKind, config) {
+    if (requestedKind == null) {
+        return null;
+    }
+    // Initial replication bypasses the interval and never runs full compaction,
+    // but small buckets should still use the normal bulk-rescheduling path.
+    if (requestedKind == CompactionKind.Chunks && !config.forceChunkCompaction && !hasEnoughChunksToMerge(state)) {
+        return null;
+    }
+    const maxOpId = config.maxOpIdCap == null || state.last_op < config.maxOpIdCap ? state.last_op : config.maxOpIdCap;
+    return state.compacted_state == null || state.compacted_state.op_id < maxOpId ? requestedKind : null;
+}
+function hasEnoughChunksToMerge(state) {
+    return state.bucket_stats.chunks - (state.compacted_state?.chunks ?? 0) >= MERGE_CHUNKS_THRESHOLD;
+}
+/**
+ * Read one bounded prefix from a compaction cursor.
+ *
+ * The document that would cross the byte limit is deliberately not returned:
+ * pagination resumes past the last returned `_id`, so that document remains
+ * eligible for the next query. The first document is always accepted to
+ * ensure progress when a single document exceeds the configured byte limit.
+ *
+ * `hasMore` is conservative when the document limit is reached. An extra empty
+ * query is preferable to exhausting the cursor just to determine whether the
+ * limited MongoDB query contained another document.
+ */
+export async function readCompactionBatch(cursor, options) {
+    const documents = [];
+    let cumulativeBytes = 0;
+    try {
+        for await (const document of cursor) {
+            if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) {
+                return { documents, hasMore: true };
+            }
+            documents.push(document);
+            cumulativeBytes += document.size;
+            if (documents.length >= options.documentLimit) {
+                return { documents, hasMore: true };
+            }
+        }
+        return { documents, hasMore: false };
+    }
+    finally {
+        await cursor.close();
+    }
+}
+//# sourceMappingURL=compact-utils.js.map

package/dist/storage/implementation/v3/compact-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compact-utils.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/compact-utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAgB,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,oBAAoB,EAAmB,MAAM,sBAAsB,CAAC;AAG7E;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAEjC,MAAM,CAAC,MAAM,iCAAiC,GAAG,EAAE,GAAG,IAAI,CAAC;AAuB3D,MAAM,CAAN,IAAY,cAGX;AAHD,WAAY,cAAc;IACxB,+BAAa,CAAA;IACb,mCAAiB,CAAA;AACnB,CAAC,EAHW,cAAc,KAAd,cAAc,QAGzB;AAcD,MAAM,OAAO,iBAAiB;IAEjB;IACA;IACA;IACA;IACA;IALX,YACW,KAAsB,EACtB,IAAoB,EACpB,QAA4B,EAC5B,mBAAqC,EACrC,QAAsB;QAJtB,UAAK,GAAL,KAAK,CAAiB;QACtB,SAAI,GAAJ,IAAI,CAAgB;QACpB,aAAQ,GAAR,QAAQ,CAAoB;QAC5B,wBAAmB,GAAnB,mBAAmB,CAAkB;QACrC,aAAQ,GAAR,QAAQ,CAAc;IAC9B,CAAC;IAEJ,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF;AAoBD,MAAM,UAAU,WAAW,CAAC,KAA4B;IACtD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK;QAC/B,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK;QAC/B,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM;KAClC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,QAAmE;IAEnE,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC5B,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,SAA8E;IAE9E,IAAI,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,qBAAqB,CAAC,KAA4B;IAChE,IAAI,KAAK,CAAC,uBAAuB,IAAI,IAAI,EAAE,CAAC;QAC1C,MAAM,IAAI,yBAAyB,CAAC,uBAAuB,KAAK,CAAC,GAAG,CAAC,CAAC,iCAAiC,CAAC,CAAC;IAC3G,CAAC;IACD,OAAO,KAAK,CAAC,uBAAuB,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,QAA+D,EAC/D,aAAsC,EACtC,yBAAkD;IAElD,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,KAAK,GAAG,yBAAyB,CAAC,KAAK,GAAG,aAAa,CAAC,KAAK;QAC7E,KAAK,EAAE,QAAQ,CAAC,KAAK,GAAG,yBAAyB,CAAC,KAAK,GAAG,aAAa,CAAC,KAAK;QAC7E,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,MAAM;QAClD,QAAQ,EAAE,YAAY,CACpB,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,yBAAyB,CAAC,QAAQ,CAAC,EAC5E,aAAa,CAAC,QAAQ,CACvB;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,KAA8B,EAC9B,MAA+B;IAE/B,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK;QACjC,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK;QACjC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QACpC,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,KAAkB,EAClB,MAA+B,EAC/B,KAA8B;IAE9B,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK;QAC/C,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK;QAC/C,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;KACpD,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,qBAAqB,CACnC,KAA8B,EAC9B,MAA+B,EAC/B,KAA8B;IAE9B,OAAO;QACL,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC;QAC7C,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC;KACvF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAA4B;IAClE,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,uBAAuB,EAAE,KAAK,CAAC,uBAAuB,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;QAC5E,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;QAC5D,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;QAChE,GAAG,oBAAoB;KACxB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAA4B,EAAE,MAA6B;IAC/F,MAAM,UAAU,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC;IAEzC,wDAAwD;IACxD,gGAAgG;IAChG,+EAA+E;IAC/E,MAAM,gBAAgB,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpG,MAAM,aAAa,GAAG,QAAQ,EAAE,IAAI,IAAI,CAAC,CAAC;IAE1C,iGAAiG;IACjG,8IAA8I;IAC9I,4GAA4G;IAC5G,mFAAmF;IACnF,MAAM,KAAK,GAAG,gBAAgB,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,CAAC,aAAa,GAAG,gBAAgB,CAAC,CAAC;IAChG,MAAM,cAAc,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC,CAAC,MAAM,CAAC,wBAAwB,GAAG,KAAK,CAAC;IAC9G,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,wBAAwB,CAAC,CAAC,CAAC;AACpG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,KAA4B,EAC5B,GAAS,EACT,MAA6B;IAE7B,gEAAgE;IAChE,4GAA4G;IAC5G,yEAAyE;IACzE,0BAA0B;IAC1B,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACzD,yEAAyE;IACzE,+DAA+D;IAC/D,MAAM,mBAAmB,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,iCAAiC,CAAC,CAAC;IAChG,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC;IACxC,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC1D,MAAM,cAAc,GAClB,SAAS,IAAI,IAAI,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,MAAM,CAAC,yBAAyB,CAAC;IAClG,6EAA6E;IAC7E,oFAAoF;IACpF,MAAM,gBAAgB,GAAmB,CAAC,mBAAmB;QAC3D,CAAC,CAAC,mBAAmB;QACrB,CAAC,CAAC;YACE,IAAI,EAAE;gBACJ,mBAAmB;gBACnB,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,CAAC,yBAAyB,EAAE,EAAE;aACpG;SACF,CAAC;IACN,IAAI,IAAI,GAA0B,IAAI,CAAC;IACvC,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;QACvB,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;IAC7B,CAAC;SAAM,IAAI,cAAc,IAAI,mBAAmB,EAAE,CAAC;QACjD,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;IAC/B,CAAC;IACD,OAAO;QACL,IAAI;QACJ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,KAA4B,EAC5B,aAAyC,EACzC,MAA2B;IAE3B,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,4EAA4E;IAC5E,wEAAwE;IACxE,IAAI,aAAa,IAAI,cAAc,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7G,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;IACnH,OAAO,KAAK,CAAC,eAAe,IAAI,IAAI,IAAI,KAAK,CAAC,eAAe,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;AACvG,CAAC;AAED,SAAS,sBAAsB,CAAC,KAA4B;IAC1D,OAAO,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,CAAC,IAAI,sBAAsB,CAAC;AACpG,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,MAAqD,EACrD,OAAqD;IAErD,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,MAAM,EAAE,CAAC;YACpC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,GAAG,QAAQ,CAAC,IAAI,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;gBAChF,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACtC,CAAC;YAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACzB,eAAe,IAAI,QAAQ,CAAC,IAAI,CAAC;YAEjC,IAAI,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC9C,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACtC,CAAC;QACH,CAAC;QACD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC"}

package/dist/storage/implementation/v3/compaction-constants.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Writers use this delay before scheduling their first chunk compaction.
+ * Initial replication includes the same delay when it picks up that work.
+ */
+export declare const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS: number;

package/dist/storage/implementation/v3/compaction-constants.js

@@ -0,0 +1,6 @@
+/**
+ * Writers use this delay before scheduling their first chunk compaction.
+ * Initial replication includes the same delay when it picks up that work.
+ */
+export const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS = 5 * 60 * 1000;
+//# sourceMappingURL=compaction-constants.js.map

package/dist/storage/implementation/v3/compaction-constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-constants.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/compaction-constants.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC"}

package/dist/storage/implementation/v3/models.d.ts

@@ -1,7 +1,7 @@
-import { InternalOpId, PersistedDefinitionMapping, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
-import { BucketDefinitionId, ParameterIndexId, ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules';
+import { InternalOpId, JsonValue, PersistedDefinitionMapping, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
+import { BucketDefinitionId, EventDefinitionId, ParameterIndexId, ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import { BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, OpType, ReplicaId, SourceTableDocumentSnapshotStatus, SourceTableKey, SyncRuleCheckpointFields, SyncRuleDocumentBase, TaggedBucketParameterDocument } from '../models.js';
+import { BucketDataKey, BucketParameterDocumentBase, CurrentBucket, OpType, ReplicaId, SourceTableDocumentSnapshotStatus, SourceTableKey, SyncRuleCheckpointFields, SyncRuleDocumentBase, TaggedBucketParameterDocument } from '../models.js';
 /**
  * Embedded in sync_rules.sync_configs.
  */
@@ -63,7 +63,7 @@ export interface SyncConfigDefinition {
     created_at: Date;
     storage_version: number;
     /**
-     * The related SyncRuleDocumentV3.
+     * The related ReplicationStreamDocumentV3.
      *
      * Note that a specific sync config definition never moves between replication streams. Instead, we can create a new copy for the new replication stream.
      *
@@ -71,6 +71,8 @@ export interface SyncConfigDefinition {
      */
     replication_stream_id: number;
     content: string;
+    /** Optional operator-supplied label identifying this sync config version. */
+    version_label?: string;
     serialized_plan?: SerializedSyncPlan | null;
     rule_mapping: PersistedDefinitionMapping;
 }
@@ -113,18 +115,68 @@ export interface SourceTableDocumentV3 {
     snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
     bucket_data_source_ids: BucketDefinitionId[];
     parameter_lookup_source_ids: ParameterIndexId[];
+    /**
+     * Stream-local ids of the compiled event definitions evaluated by this source table.
+     */
+    event_definition_ids: EventDefinitionId[];
     latest_pending_delete?: InternalOpId | undefined;
+    /**
+     * Source-specific metadata. Absent for legacy records.
+     */
+    source_metadata?: JsonValue;
 }
-export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
-    _id: BucketStateDocumentBase['_id'] & {
+export interface BucketStateDocumentV3 {
+    _id: {
+        b: string;
         d: BucketDefinitionId;
     };
+    /** Must always identify an actual operation in this logical stream. */
+    last_op: bigint;
+    /** The next time a compact worker should inspect this bucket. */
+    next_compact_check: Date | undefined;
+    /**
+     * Scheduling epoch for work not covered by a full compact. Writers set the
+     * first actual write time; a partial full compact advances it to completion.
+     */
+    first_uncompacted_write: Date | undefined;
+    /**
+     * A checksum cache and the statistics captured by the latest compact (full
+     * or chunk). Keeping these separate from bucket_stats lets writers only
+     * update one set of counters.
+     */
+    compacted_state?: {
+        op_id: InternalOpId;
+        checksum: bigint;
+        count: number;
+        bytes: bigint;
+        at: Date;
+        chunks: number;
+    };
+    /** Statistics for the prefix covered by the most recent full compact. */
+    last_full_compact?: {
+        op_id: InternalOpId;
+        count: number;
+        at: Date;
+        puts: number;
+    };
+    /** Current aggregate bucket statistics, maintained by writers and compactors. */
+    bucket_stats: {
+        count: number;
+        bytes: bigint;
+        chunks: number;
+    };
+    /** A short-lived ownership marker used to distribute bucket compaction. */
+    compact_lease?: {
+        expires_at: Date;
+        id: unknown;
+    };
 }
 export interface BucketOperation {
     o: bigint;
     op: OpType;
     source_table?: bson.ObjectId;
     source_key?: ReplicaId;
+    subkey?: string;
     table?: string;
     row_id?: string;
     checksum: bigint;
@@ -134,6 +186,14 @@ export interface StorageRef {
     path: string;
     file_size: number;
 }
+export interface ObjectStorageUsageDocument {
+    _id: {
+        g: number;
+        w: string;
+    };
+    updated_at: Date;
+    definitions: Record<BucketDefinitionId, bigint>;
+}
 /** An S3 object that may be deleted once its grace period has elapsed. */
 export interface ObjectStorageDeletionMarker {
     _id: bson.ObjectId;
@@ -176,6 +236,30 @@ export interface BucketDataDocumentV3 {
      */
     has_clear_op?: true;
 }
+/**
+ * v3 Custom checkpoints (new name for legacy write checkpoints) are related to
+ * a replication stream and a specific event (in a sync config) which created them.
+ * Unchanged events, between sync configs, can share the documents.
+ * For v3, we store each replication stream's custom checkpoint requests in their own collection:
+ * `custom_checkpoint_requests_${replicationStreamId}_${eventId}`
+ */
+export interface CustomCheckpointRequestDocumentV3 {
+    _id: bson.ObjectId;
+    user_id: string;
+    checkpoint: bigint;
+    /**
+     * Unlike managed write checkpoints, custom write checkpoints are flushed together with
+     * normal ops. This means we can assign an op_id for ordering / correlating with read checkpoints.
+     *
+     * This is not unique - multiple write checkpoints can have the same op_id.
+     */
+    op_id?: InternalOpId;
+    /**
+     * Set when this checkpoint was created from a client-supplied checkpoint
+     * request rather than a persistent custom write checkpoint.
+     */
+    checkpoint_requested_at?: Date | null;
+}
 export declare function serializeParameterLookup(lookup: ScopedParameterLookup): bson.Binary;
 export declare function deserializeParameterLookup(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[];
 export declare function taggedBucketParameterDocumentToTagged(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;

package/dist/storage/implementation/v3/models.js.map

@@ -1 +1 @@
-{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,0BAA0B,IAAI,8BAA8B,EAC7D,MAAM,yBAAyB,CAAC;AAOjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AA2H7B,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC;AAwFD,MAAM,UAAU,wBAAwB,CAAC,MAA6B;IACpE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAAmB,EAAE,OAAyB;IACvF,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,8BAA8B,CAAC,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,qCAAqC,CACnD,QAAuC;IAEvC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,0BAA0B,IAAI,8BAA8B,EAC7D,MAAM,yBAAyB,CAAC;AAQjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AA4H7B,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC;AAkLD,MAAM,UAAU,wBAAwB,CAAC,MAA6B;IACpE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAAmB,EAAE,OAAyB;IACvF,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,8BAA8B,CAAC,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,qCAAqC,CACnD,QAAuC;IAEvC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}

package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts

@@ -1,21 +1,20 @@
+import { PerformanceTracer } from '@powersync/service-core';
 import { BucketDataDocumentV3, BucketOperation } from '../models.js';
-import { ObjectStorage } from './ObjectStorage.js';
+import { ObjectStorage, ObjectStorageOperationOptions } from './ObjectStorage.js';
 export declare class BucketDataObjectStorage {
     private readonly storage;
     constructor(storage: ObjectStorage);
-    store(path: string, ops: BucketOperation[]): Promise<{
+    store(path: string, ops: BucketOperation[], options?: ObjectStorageOperationOptions): Promise<{
         fileSize: number;
     }>;
-    retrieve(path: string, options: {
-        signal?: AbortSignal;
-    }): Promise<BucketOperation[]>;
-    delete(paths: string[]): Promise<void>;
+    retrieve(path: string, options: ObjectStorageOperationOptions): Promise<BucketOperation[]>;
+    delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void>;
 }
 /**
  * Load offloaded operations and patch them onto their MongoDB metadata documents.
  * The object-storage implementation is responsible for limiting request
  * concurrency across all callers.
  */
-export declare function hydrateBucketDataDocuments(documents: BucketDataDocumentV3[], objectStorage: ObjectStorage | undefined, options: {
-    signal?: AbortSignal;
+export declare function hydrateBucketDataDocuments(documents: BucketDataDocumentV3[], objectStorage: ObjectStorage | undefined, options: ObjectStorageOperationOptions & {
+    tracer?: PerformanceTracer<string>;
 }): Promise<void>;

package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js

@@ -5,9 +5,9 @@ export class BucketDataObjectStorage {
     constructor(storage) {
         this.storage = storage;
     }
-    async store(path, ops) {
+    async store(path, ops, options) {
         const bsonBuffer = bson.serialize({ ops });
-        await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null });
+        await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null }, options);
         return { fileSize: bsonBuffer.byteLength };
     }
     async retrieve(path, options) {
@@ -21,8 +21,8 @@ export class BucketDataObjectStorage {
         const wrapper = bson.deserialize(data, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
         return wrapper.ops;
     }
-    async delete(paths) {
-        return this.storage.delete(paths);
+    async delete(paths, options) {
+        return this.storage.delete(paths, options);
     }
 }
 /**
@@ -37,8 +37,19 @@ export async function hydrateBucketDataDocuments(documents, objectStorage, optio
     options.signal?.throwIfAborted();
     const store = new BucketDataObjectStorage(objectStorage);
     const storedDocuments = documents.filter((document) => document.storage_ref);
-    await Promise.all(storedDocuments.map(async (document) => {
+    if (storedDocuments.length == 0) {
+        return;
+    }
+    using _ = options?.tracer?.span('s3', 'read');
+    const results = await Promise.allSettled(storedDocuments.map(async (document) => {
         document.ops = await store.retrieve(document.storage_ref.path, { signal: options.signal });
     }));
+    // Drain every download before propagating failure so callers can release their
+    // worker slot and payloads without a sibling download hydrating a document later.
+    for (const result of results) {
+        if (result.status === 'rejected') {
+            throw result.reason;
+        }
+    }
 }
 //# sourceMappingURL=BucketDataObjectStorage.js.map

package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map

@@ -1 +1 @@
-{"version":3,"file":"BucketDataObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAI7B,MAAM,OAAO,uBAAuB;IACL;IAA7B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;IAAG,CAAC;IAEvD,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,GAAsB;QAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACrG,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,OAAiC;QAC5D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACpF,IAAI,QAAQ,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,QAAQ,CAAC,WAAW,KAAK,kBAAkB,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAClF,OAAO,OAAO,CAAC,GAAG,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,SAAiC,EACjC,aAAwC,EACxC,OAAiC;IAEjC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,uBAAuB,CAAC,aAAa,CAAC,CAAC;IACzD,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC7E,MAAM,OAAO,CAAC,GAAG,CACf,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QACrC,QAAQ,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAY,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"BucketDataObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAI7B,MAAM,OAAO,uBAAuB;IACL;IAA7B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;IAAG,CAAC;IAEvD,KAAK,CAAC,KAAK,CACT,IAAY,EACZ,GAAsB,EACtB,OAAuC;QAEvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QAC9G,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,OAAsC;QACjE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACpF,IAAI,QAAQ,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,QAAQ,CAAC,WAAW,KAAK,kBAAkB,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAClF,OAAO,OAAO,CAAC,GAAG,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe,EAAE,OAAuC;QACnE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,SAAiC,EACjC,aAAwC,EACxC,OAA+E;IAE/E,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,uBAAuB,CAAC,aAAa,CAAC,CAAC;IACzD,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC7E,IAAI,eAAe,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,OAAO;IACT,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QACrC,QAAQ,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAY,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CACH,CAAC;IACF,+EAA+E;IAC/E,kFAAkF;IAClF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts

@@ -10,21 +10,24 @@ export declare class ObjectStorageError extends Error {
     });
 }
 export declare function isRetryableObjectStorageError(error: unknown): error is ObjectStorageError;
+export interface ObjectStorageOperationOptions {
+    /**
+     * Cancels the operation, including while it waits for a concurrency slot.
+     *
+     * Operations are bounded by their own timeouts regardless, but a signal releases the slot as
+     * soon as the caller no longer needs the result.
+     */
+    signal?: AbortSignal;
+}
 export interface ObjectStorage {
-    put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void>;
-    get(path: string, options?: {
-        signal?: AbortSignal;
-    }): Promise<{
+    put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata, options?: ObjectStorageOperationOptions): Promise<void>;
+    get(path: string, options?: ObjectStorageOperationOptions): Promise<{
         data: Uint8Array;
         metadata: ObjectStoragePutMetadata;
     }>;
-    list(prefix: string, options?: {
-        signal?: AbortSignal;
-    }): AsyncIterable<string>;
-    delete(paths: string[]): Promise<void>;
-    deletePrefix(prefix: string, options?: {
-        signal?: AbortSignal;
-    }): Promise<{
+    list(prefix: string, options?: ObjectStorageOperationOptions): AsyncIterable<string>;
+    delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void>;
+    deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{
         objectCount: number;
     }>;
 }

package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts

@@ -1,9 +1,9 @@
-import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
 import * as bson from 'bson';
+import { MongoWriteBatch } from '../../MongoWriteBatch.js';
 import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
 import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
-import { ObjectStorage } from './ObjectStorage.js';
+import { ObjectStorage, ObjectStorageOperationOptions } from './ObjectStorage.js';
 export declare const OBJECT_STORAGE_UPLOAD_LEASE_MS: number;
 export declare const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS: number;
 export declare const OBJECT_STORAGE_REFERENCE_GRACE_MS: number;
@@ -44,7 +44,7 @@ export declare class ObjectStorageLifecycle {
      *
      * Call this in the same transaction as the one creating the references.
      */
-    publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void>;
+    publishUploads(uploads: PreparedObjectStorageUpload[], writes: MongoWriteBatch): void;
     /**
      * Mark old paths for deletion.
      *
@@ -56,15 +56,13 @@ export declare class ObjectStorageLifecycle {
      *
      * Call this in the same transaction as the one removing the references to these files.
      */
-    retire(paths: Iterable<string>, session: mongo.ClientSession, now?: Date): Promise<void>;
+    retire(paths: Iterable<string>, writes: MongoWriteBatch, now?: Date): void;
     /**
      * Discover and delete objects by their storage prefix
      * without touching the database.
      */
-    deletePrefix(prefix: string, options?: {
-        signal?: AbortSignal;
-    }): Promise<{
+    deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{
         objectCount: number;
     }>;
-    cleanup(logger: Logger): Promise<void>;
+    cleanup(logger: Logger, options?: ObjectStorageOperationOptions): Promise<void>;
 }

package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js

@@ -72,7 +72,7 @@ export class ObjectStorageLifecycle {
      *
      * Call this in the same transaction as the one creating the references.
      */
-    async publishUploads(uploads, session) {
+    publishUploads(uploads, writes) {
         for (const upload of uploads) {
             if (!this.canPublish(upload)) {
                 throw new Error(`Object storage publication lease expired for ${upload.path}`);
@@ -81,12 +81,10 @@ export class ObjectStorageLifecycle {
         if (uploads.length === 0) {
             return;
         }
-        const result = await this.db
-            .pendingObjectStorageDeletes(this.replicationStreamId)
-            .deleteMany({ _id: { $in: uploads.map((upload) => upload.markerId) } }, { session });
-        if (result.deletedCount !== uploads.length) {
-            throw new Error(`Missing object storage publication markers`);
-        }
+        writes.deleteMany(this.db.pendingObjectStorageDeletes(this.replicationStreamId), { _id: { $in: uploads.map((upload) => upload.markerId) } }, (deletedCount) => {
+            if (deletedCount !== uploads.length)
+                throw new Error('Missing object storage publication markers');
+        });
     }
     /**
      * Mark old paths for deletion.
@@ -99,14 +97,14 @@ export class ObjectStorageLifecycle {
      *
      * Call this in the same transaction as the one removing the references to these files.
      */
-    async retire(paths, session, now = new Date()) {
+    retire(paths, writes, now = new Date()) {
         const markers = Array.from(paths, (path) => ({
             _id: new bson.ObjectId(),
             path,
             delete_after: new Date(now.getTime() + OBJECT_STORAGE_REFERENCE_GRACE_MS)
         }));
         if (markers.length) {
-            await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(markers, { session });
+            writes.insertMany(this.db.pendingObjectStorageDeletes(this.replicationStreamId), markers);
         }
     }
     /**
@@ -116,7 +114,8 @@ export class ObjectStorageLifecycle {
     async deletePrefix(prefix, options) {
         return this.objectStorage.deletePrefix(prefix, { signal: options?.signal });
     }
-    async cleanup(logger) {
+    async cleanup(logger, options) {
+        const signal = options?.signal;
         const markers = this.db.pendingObjectStorageDeletes(this.replicationStreamId);
         const cursor = markers.find({ delete_after: { $lte: new Date() } });
         let batch = [];
@@ -127,14 +126,21 @@ export class ObjectStorageLifecycle {
             const deleting = batch;
             batch = [];
             try {
-                await this.bucketData.delete(deleting.map((marker) => marker.path));
+                await this.bucketData.delete(deleting.map((marker) => marker.path), { signal });
                 await markers.deleteMany({ _id: { $in: deleting.map((marker) => marker._id) } });
             }
             catch (error) {
+                if (signal?.aborted) {
+                    // Shutting down: stop instead of working through the remaining markers. They stay
+                    // pending, and are cleaned up during the next compaction. Checked on the signal rather
+                    // than the error, since a caller may abort with any reason.
+                    throw error;
+                }
                 logger.warn('Failed to clean up object storage deletion markers; will retry during the next compaction', error);
             }
         };
         for await (const marker of cursor) {
+            signal?.throwIfAborted();
             batch.push(marker);
             if (batch.length === 500) {
                 await deleteBatch();