@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

package/src/storage/implementation/v1/PersistedBatchV1.ts

@@ -6,6 +6,7 @@ import * as bson from 'bson';
 import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
 import { mongoTableId } from '../../../utils/util.js';
 import { EMPTY_DATA } from '../MongoBucketBatchShared.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import {
   BucketStateUpdate,
   PersistedBatch,
@@ -25,7 +26,9 @@ import {
 export class PersistedBatchV1 extends PersistedBatch {
   declare protected readonly db: VersionedPowerSyncMongoV1;
 
-  currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[] = [];
+  // Each upsert supplies the complete source-record state. Only its final
+  // state matters within a flush; bucket and parameter history is kept separately.
+  currentData = new Map<string, mongo.AnyBulkWriteOperation<CurrentDataDocument>>();
 
   protected checkDefinitionId(_definitionId: BucketDefinitionId | null): BucketDefinitionId {
     // V1 storage doesn't persist the id, and we don't use it.
@@ -94,7 +97,7 @@ export class PersistedBatchV1 extends PersistedBatch {
   }
 
   hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) {
-    this.currentData.push({
+    this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
       deleteOne: {
         filter: { _id: this.currentDataId(sourceTableId, replicaId) }
       }
@@ -124,7 +127,7 @@ export class PersistedBatchV1 extends PersistedBatch {
       return lookup.lookup;
     });
 
-    this.currentData.push({
+    this.currentData.set(this.currentDataKey(values.sourceTableId, values.replicaId), {
       updateOne: {
         filter: { _id: this.currentDataId(values.sourceTableId, values.replicaId) },
         update: {
@@ -141,57 +144,45 @@ export class PersistedBatchV1 extends PersistedBatch {
   }
 
   protected get currentDataCount() {
-    return this.currentData.length;
+    return this.currentData.size;
   }
 
-  protected async flushBucketData(session: mongo.ClientSession) {
-    await this.db.bucketDataV1.bulkWrite(
+  protected async queueBucketData(writes: MongoWriteBatch) {
+    writes.bulkWriteUnordered(
+      this.db.bucketDataV1,
       this.bucketData.map((document) => ({
         insertOne: {
           document: serializeBucketDataV1(document)
         }
-      })),
-      {
-        session,
-        ordered: false
-      }
+      }))
     );
   }
 
-  protected async flushBucketParameters(session: mongo.ClientSession) {
-    await this.db.parameterIndexV1.bulkWrite(
+  protected queueBucketParameters(writes: MongoWriteBatch): void {
+    writes.bulkWriteUnordered(
+      this.db.parameterIndexV1,
       this.bucketParameters.map((document) => ({
         insertOne: {
           document: taggedBucketParameterDocumentToV1(document)
         }
-      })),
-      {
-        session,
-        ordered: false
-      }
+      }))
     );
   }
 
-  protected async flushCurrentData(session: mongo.ClientSession) {
-    if (this.currentData.length == 0) {
+  protected queueCurrentData(writes: MongoWriteBatch): void {
+    if (this.currentData.size == 0) {
       return;
     }
 
-    await this.db.sourceRecordsV1.bulkWrite(this.currentData, {
-      session,
-      ordered: true
-    });
+    writes.bulkWriteUnordered(this.db.sourceRecordsV1, [...this.currentData.values()]);
   }
 
-  protected async flushBucketStates(session: mongo.ClientSession) {
-    await this.db.bucketStateV1.bulkWrite(this.getBucketStateUpdates(), {
-      session,
-      ordered: false
-    });
+  protected queueBucketStates(writes: MongoWriteBatch): void {
+    writes.bulkWriteUnordered(this.db.bucketStateV1, this.getBucketStateUpdates());
   }
 
   protected resetCurrentData() {
-    this.currentData = [];
+    this.currentData.clear();
   }
 
   private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocumentV1>[] {
@@ -219,6 +210,10 @@ export class PersistedBatchV1 extends PersistedBatch {
     });
   }
 
+  private currentDataKey(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): string {
+    return Buffer.from(bson.serialize({ t: sourceTableId, k: replicaId })).toString('base64');
+  }
+
   private currentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey {
     return {
       g: this.group_id,

package/src/storage/implementation/v1/models.ts

@@ -48,6 +48,7 @@ export function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocume
     op: document.op,
     source_table: document.source_table,
     source_key: document.source_key,
+    subkey: document.subkey,
     table: document.table,
     row_id: document.row_id,
     checksum: document.checksum,

package/src/storage/implementation/v3/CompactionLease.ts

@@ -0,0 +1,152 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { InternalOpId } from '@powersync/service-core';
+import { CompactionLeaseLostError } from '../MongoCompactor.js';
+import { BucketStateDocumentV3 } from './models.js';
+
+const LEASE_RENEW_INTERVAL_MS = 60 * 1000;
+
+/**
+ * Filters for buckets that have no lease or an expired lease.
+ */
+export const AVAILABLE_LEASE_EXPR = {
+  // $$NOW is evaluated by MongoDB, avoiding lease expiry races due
+  // to clocks on separate compact workers.
+  $expr: {
+    $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }]
+  }
+};
+
+/**
+ * Owns one V3 bucket-compaction lease, including its server-time renewal and
+ * the owner-fenced operations which release it.
+ *
+ * This is intended as a way to reduce redundant work if multiple jobs attempt to
+ * compact the same bucket concurrently, but it's not an absolute safety guarantee.
+ * The individual operations must still be designed to be safe with concurrent compacting.
+ */
+export class CompactionLease implements AsyncDisposable {
+  readonly startedAt: Date;
+  readonly lastOp: InternalOpId;
+
+  private timer: NodeJS.Timeout | undefined;
+  private renewalInFlight = false;
+  private renewalError: unknown;
+  private finalizing = false;
+  private finished = false;
+
+  private constructor(
+    private readonly collection: mongo.Collection<BucketStateDocumentV3>,
+    readonly state: BucketStateDocumentV3,
+    readonly id: mongo.ObjectId,
+    private readonly durationMs: number
+  ) {
+    this.startedAt = new Date(state.compact_lease!.expires_at.getTime() - durationMs);
+    this.lastOp = state.last_op;
+  }
+
+  static async claim(
+    collection: mongo.Collection<BucketStateDocumentV3>,
+    filter: mongo.Filter<BucketStateDocumentV3>,
+    sort: mongo.Sort | undefined,
+    durationMs: number
+  ): Promise<CompactionLease | null> {
+    const id = new mongo.ObjectId();
+    const state = await collection.findOneAndUpdate(
+      {
+        $and: [filter, AVAILABLE_LEASE_EXPR]
+      },
+      [
+        {
+          $set: {
+            compact_lease: {
+              id,
+              expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: durationMs } }
+            }
+          }
+        }
+      ],
+      { sort, returnDocument: 'after' }
+    );
+    return state == null ? null : new CompactionLease(collection, state, id, durationMs);
+  }
+
+  startRenewal() {
+    const interval = Math.max(1, Math.min(LEASE_RENEW_INTERVAL_MS, Math.floor(this.durationMs / 2)));
+    this.timer = setInterval(() => {
+      if (this.renewalInFlight) {
+        return;
+      }
+      this.renewalInFlight = true;
+      void this.renew()
+        .catch((error) => {
+          this.renewalError = error;
+        })
+        .finally(() => {
+          this.renewalInFlight = false;
+        });
+    }, interval);
+    this.timer.unref();
+  }
+
+  async throwIfLost() {
+    if (this.renewalError != null) {
+      throw this.renewalError;
+    }
+  }
+
+  async reschedule(nextCompactCheck: mongo.Document) {
+    await this.finish([{ $set: { next_compact_check: nextCompactCheck } }, { $unset: 'compact_lease' }]);
+  }
+
+  async finalize(update: mongo.Document) {
+    await this.finish([{ $set: update }, { $unset: 'compact_lease' }]);
+  }
+
+  async [Symbol.asyncDispose]() {
+    if (this.timer != null) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    if (this.finished) {
+      return;
+    }
+    this.finalizing = true;
+    await this.collection.updateOne(this.filter, { $unset: { compact_lease: '' } });
+    this.finished = true;
+  }
+
+  private get filter(): mongo.Filter<BucketStateDocumentV3> {
+    return { _id: this.state._id, 'compact_lease.id': this.id };
+  }
+
+  private async renew() {
+    const result = await this.collection.updateOne(this.filter, [
+      {
+        $set: {
+          'compact_lease.expires_at': {
+            $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.durationMs }
+          }
+        }
+      }
+    ]);
+    // A finalization in progress owns the authoritative fenced result.
+    if (result.matchedCount != 1 && !this.finalizing) {
+      throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`);
+    }
+    // A successful renewal confirms that a transient failure has passed. A
+    // lease-loss error remains sticky: it means another worker may own it.
+    if (!(this.renewalError instanceof CompactionLeaseLostError)) {
+      this.renewalError = undefined;
+    }
+  }
+
+  private async finish(update: mongo.Document[]) {
+    await this.throwIfLost();
+    this.finalizing = true;
+    const result = await this.collection.updateOne(this.filter, update);
+    if (result.matchedCount != 1) {
+      throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`);
+    }
+    this.finished = true;
+  }
+}