@powersync/service-module-mongodb-storage 0.21.0 → 0.22.0

package/src/storage/implementation/v3/PersistedBatchV3.ts

@@ -13,6 +13,7 @@ import {
   UpsertCurrentDataOptions
 } from '../common/PersistedBatch.js';
 import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import { serializeBucketData } from './bucket-format.js';
 import { chunkBucketData } from './chunking.js';
 import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js';
@@ -28,8 +29,15 @@ import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.
 import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
+interface SourceRecordWrite {
+  sourceTableId: bson.ObjectId;
+  operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3>;
+}
+
 export class PersistedBatchV3 extends PersistedBatch {
-  currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
+  // Upserts and soft deletes supply the complete source-record state, including
+  // pending_delete. Keep the final state per key, without removing any history.
+  currentData = new Map<string, SourceRecordWrite>();
   sourceTablePendingDeletes = new Map<string, InternalOpId>();
   protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
   protected readonly objectStorageUsage?: ObjectStorageUsage;
@@ -131,7 +139,7 @@ export class PersistedBatchV3 extends PersistedBatch {
   }
 
   hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) {
-    this.currentData.push({
+    this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
       sourceTableId,
       operation: {
         deleteOne: {
@@ -147,7 +155,7 @@ export class PersistedBatchV3 extends PersistedBatch {
     replicaId: storage.ReplicaId,
     checkpointGreaterThan: InternalOpId
   ) {
-    this.currentData.push({
+    this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
       sourceTableId,
       operation: {
         updateOne: {
@@ -195,7 +203,7 @@ export class PersistedBatchV3 extends PersistedBatch {
       };
     });
 
-    this.currentData.push({
+    this.currentData.set(this.currentDataKey(values.sourceTableId, values.replicaId), {
       sourceTableId: values.sourceTableId,
       operation: {
         updateOne: {
@@ -216,12 +224,12 @@ export class PersistedBatchV3 extends PersistedBatch {
   }
 
   protected get currentDataCount() {
-    return this.currentData.length;
+    return this.currentData.size;
   }
 
   // Flush methods
 
-  protected async flushBucketData(session: mongo.ClientSession) {
+  protected async queueBucketData(writes: MongoWriteBatch) {
     const operationsByDefinition = new Map<BucketDefinitionId, BucketDataDoc[]>();
     for (const document of this.bucketData) {
       const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
@@ -284,10 +292,10 @@ export class PersistedBatchV3 extends PersistedBatch {
     // S3ObjectStorage applies one shared concurrency limit across all callers,
     // so replication can schedule its uploads together without creating a
     // separate limiter here.
-    const writes = await createAllInserts();
+    const bucketWrites = await createAllInserts();
 
     const usageDeltas = this.objectStorageUsage == null ? undefined : new Map<BucketDefinitionId, bigint>();
-    for (const { definitionId, inserts } of writes) {
+    for (const { definitionId, inserts } of bucketWrites) {
       if (usageDeltas != null) {
         const delta = inserts.reduce(
           (sum, operation) => sum + ObjectStorageUsage.bytes(operation.insertOne.document),
@@ -298,18 +306,15 @@ export class PersistedBatchV3 extends PersistedBatch {
         }
       }
       if (inserts.length > 0) {
-        await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
-          session,
-          ordered: false
-        });
+        writes.bulkWriteUnordered(this.db.bucketData(this.group_id, definitionId), inserts);
       }
     }
     if (usageDeltas != null) {
-      await this.objectStorageUsage!.applyDeltas(usageDeltas, session);
+      this.objectStorageUsage!.applyDeltas(usageDeltas, writes);
     }
   }
 
-  protected async flushBucketParameters(session: mongo.ClientSession) {
+  protected queueBucketParameters(writes: MongoWriteBatch): void {
     const operationsByIndex = new Map<string, typeof this.bucketParameters>();
     for (const document of this.bucketParameters) {
       const existing = operationsByIndex.get(document.index) ?? [];
@@ -318,23 +323,20 @@ export class PersistedBatchV3 extends PersistedBatch {
     }
 
     for (const [indexId, documents] of operationsByIndex.entries()) {
-      await this.db.parameterIndex(this.group_id, indexId).bulkWrite(
+      writes.bulkWriteUnordered(
+        this.db.parameterIndex(this.group_id, indexId),
         documents.map((document) => ({
           insertOne: {
             document: taggedBucketParameterDocumentToTagged(document)
           }
-        })),
-        {
-          session,
-          ordered: false
-        }
+        }))
       );
     }
   }
 
-  protected async flushCurrentData(session: mongo.ClientSession) {
-    const operationsBySourceTable = new Map<string, typeof this.currentData>();
-    for (const operation of this.currentData) {
+  protected queueCurrentData(writes: MongoWriteBatch): void {
+    const operationsBySourceTable = new Map<string, SourceRecordWrite[]>();
+    for (const operation of this.currentData.values()) {
       const sourceTableId = operation.sourceTableId.toHexString();
       const existing = operationsBySourceTable.get(sourceTableId) ?? [];
       existing.push(operation);
@@ -357,33 +359,31 @@ export class PersistedBatchV3 extends PersistedBatch {
     });
 
     if (sourceTableUpdates.length > 0) {
-      await this.db.sourceTables(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
+      writes.bulkWriteUnordered(this.db.sourceTables(this.group_id), sourceTableUpdates);
     }
 
     for (const operations of operationsBySourceTable.values()) {
       const sourceTableId = operations[0]!.sourceTableId;
-      await this.db.sourceRecords(this.group_id, sourceTableId).bulkWrite(
-        operations.map((entry) => entry.operation),
-        {
-          session,
-          ordered: true
-        }
+      writes.bulkWriteUnordered(
+        this.db.sourceRecords(this.group_id, sourceTableId),
+        operations.map((entry) => entry.operation)
       );
     }
   }
 
-  protected async flushBucketStates(session: mongo.ClientSession) {
-    await this.db.bucketState(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
-      session,
-      ordered: false
-    });
+  protected queueBucketStates(writes: MongoWriteBatch): void {
+    writes.bulkWriteUnordered(this.db.bucketState(this.group_id), this.getBucketStateUpdates());
   }
 
   protected resetCurrentData() {
-    this.currentData = [];
+    this.currentData.clear();
     this.sourceTablePendingDeletes.clear();
   }
 
+  private currentDataKey(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): string {
+    return Buffer.from(bson.serialize({ t: sourceTableId, k: replicaId })).toString('base64');
+  }
+
   private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocumentV3>[] {
     return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => {
       if (state.definitionId == null) {

package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts

@@ -54,9 +54,16 @@ export async function hydrateBucketDataDocuments(
     return;
   }
   using _ = options?.tracer?.span('s3', 'read');
-  await Promise.all(
+  const results = await Promise.allSettled(
     storedDocuments.map(async (document) => {
       document.ops = await store.retrieve(document.storage_ref!.path, { signal: options.signal });
     })
   );
+  // Drain every download before propagating failure so callers can release their
+  // worker slot and payloads without a sibling download hydrating a document later.
+  for (const result of results) {
+    if (result.status === 'rejected') {
+      throw result.reason;
+    }
+  }
 }

package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts

@@ -1,7 +1,7 @@
-import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
 import * as bson from 'bson';
 import { createHash, randomUUID } from 'node:crypto';
+import { MongoWriteBatch } from '../../MongoWriteBatch.js';
 import { ObjectStorageDeletionMarker } from '../models.js';
 import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
 import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
@@ -93,7 +93,7 @@ export class ObjectStorageLifecycle {
    *
    * Call this in the same transaction as the one creating the references.
    */
-  async publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void> {
+  publishUploads(uploads: PreparedObjectStorageUpload[], writes: MongoWriteBatch): void {
     for (const upload of uploads) {
       if (!this.canPublish(upload)) {
         throw new Error(`Object storage publication lease expired for ${upload.path}`);
@@ -103,12 +103,13 @@ export class ObjectStorageLifecycle {
       return;
     }
 
-    const result = await this.db
-      .pendingObjectStorageDeletes(this.replicationStreamId)
-      .deleteMany({ _id: { $in: uploads.map((upload) => upload.markerId) } }, { session });
-    if (result.deletedCount !== uploads.length) {
-      throw new Error(`Missing object storage publication markers`);
-    }
+    writes.deleteMany(
+      this.db.pendingObjectStorageDeletes(this.replicationStreamId),
+      { _id: { $in: uploads.map((upload) => upload.markerId) } },
+      (deletedCount) => {
+        if (deletedCount !== uploads.length) throw new Error('Missing object storage publication markers');
+      }
+    );
   }
 
   /**
@@ -122,14 +123,14 @@ export class ObjectStorageLifecycle {
    *
    * Call this in the same transaction as the one removing the references to these files.
    */
-  async retire(paths: Iterable<string>, session: mongo.ClientSession, now = new Date()): Promise<void> {
+  retire(paths: Iterable<string>, writes: MongoWriteBatch, now = new Date()): void {
     const markers: ObjectStorageDeletionMarker[] = Array.from(paths, (path) => ({
       _id: new bson.ObjectId(),
       path,
       delete_after: new Date(now.getTime() + OBJECT_STORAGE_REFERENCE_GRACE_MS)
     }));
     if (markers.length) {
-      await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(markers, { session });
+      writes.insertMany(this.db.pendingObjectStorageDeletes(this.replicationStreamId), markers);
     }
   }
 

package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts

@@ -3,6 +3,7 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { BucketDefinitionId } from '@powersync/service-sync-rules';
 import { randomUUID } from 'node:crypto';
+import { MongoWriteBatch } from '../../MongoWriteBatch.js';
 import { BucketDataDocumentV3, ObjectStorageUsageDocument } from '../models.js';
 import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
 
@@ -93,11 +94,11 @@ export class ObjectStorageUsage {
     });
   }
 
-  async applyDelta(definitionId: BucketDefinitionId, delta: bigint, session: mongo.ClientSession): Promise<void> {
-    await this.applyDeltas(new Map([[definitionId, delta]]), session);
+  applyDelta(definitionId: BucketDefinitionId, delta: bigint, writes: MongoWriteBatch): void {
+    this.applyDeltas(new Map([[definitionId, delta]]), writes);
   }
 
-  async applyDeltas(deltas: ReadonlyMap<BucketDefinitionId, bigint>, session: mongo.ClientSession): Promise<void> {
+  applyDeltas(deltas: ReadonlyMap<BucketDefinitionId, bigint>, writes: MongoWriteBatch): void {
     if (this.writerId == null) {
       throw new ReplicationAssertionError('A writer id is required to apply object-storage usage');
     }
@@ -113,13 +114,11 @@ export class ObjectStorageUsage {
       return;
     }
 
-    await this.db.objectStorageUsage.updateOne(
+    writes.updateOne(
+      this.db.objectStorageUsage,
       { _id: this.documentId() },
-      {
-        $inc: increments,
-        $currentDate: { updated_at: true }
-      },
-      { upsert: true, session }
+      { $inc: increments, $currentDate: { updated_at: true } },
+      { upsert: true }
     );
   }
 

package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts

@@ -19,10 +19,13 @@ import {
   type ObjectStoragePutMetadata
 } from './ObjectStorage.js';
 
-const DEFAULT_S3_OPERATION_CONCURRENCY = 16;
+// Never statically import (or re-export) this module!
+// We want to avoid loading the AWS SDK unless the user has configured S3 object storage, since it's a large dependency.
+
+const DEFAULT_S3_OPERATION_CONCURRENCY = 64;
 const S3_DELETE_PREFIX_BATCH_SIZE = 1000;
 /**
- * Slightly smaller than DEFAULT_S3_OPERATION_CONCURRENCY.
+ * Keep prefix deletion below the default shared limit to leave capacity for other operations.
  */
 const S3_DELETE_PREFIX_CONCURRENCY = 12;
 const MAX_S3_PREFIX_BYTES = 256;
@@ -54,12 +57,20 @@ const SAFE_S3_KEY_BYTES = 896;
 const BASELINE_CONNECTION_TIMEOUT_MS = 3_100;
 const REQUEST_TIMEOUT_FACTOR = 5;
 const OPERATION_TIMEOUT_FACTOR = 2;
+/**
+ * Don't keep idle http sockets around for longer than this timeout.
+ *
+ * This covers the case where we have large bursts of activity during replication, followed by
+ * long periods of inactivity.
+ */
+const SOCKET_IDLE_TIMEOUT_MS = 60_000;
+
 /**
  * How long an operation may wait for a concurrency slot, as a multiple of the operation timeout.
  *
  * Every slot holder now releases within the operation timeout, so the queue always drains: this is
  * a backpressure limit rather than a deadlock guard. Waiting this long means at least
- * `factor * concurrencyLimit` operations ahead of us each took their full deadline - 64 at the
+ * `factor * concurrencyLimit` operations ahead of us each took their full deadline - 256 at the
  * default concurrency. Real operations complete in a fraction of the deadline, so the number of
  * queued operations this actually tolerates is far higher, well above the ~2000 uploads that the
  * largest possible replication flush (MAX_TRANSACTION_DOC_COUNT) can enqueue at once.
@@ -221,6 +232,15 @@ export class S3ObjectStorage implements ObjectStorage {
       forcePathStyle: options.forcePathStyle,
       defaultsMode,
       requestHandler: new NodeHttpHandler({
+        // Match the shared request limit instead of queuing again at the HTTP pool.
+        httpAgent: {
+          keepAlive: true,
+          maxSockets: concurrencyLimit,
+          // The timeout here is a little more general than an "idle timeout", but it does cover the
+          // case of closing idle sockets after a period of inactivity.
+          timeout: SOCKET_IDLE_TIMEOUT_MS
+        },
+        httpsAgent: { keepAlive: true, maxSockets: concurrencyLimit, timeout: SOCKET_IDLE_TIMEOUT_MS },
         connectionTimeout: this.timeouts.connectionTimeoutMs,
         requestTimeout: this.timeouts.requestTimeoutMs,
         // Without this, exceeding requestTimeout only logs a warning.

package/src/types/types.ts

@@ -65,6 +65,9 @@ export const MongoStorageConfig = service_types.configFile.BaseStorageConfig.and
      */
     clear_batch_throttle_rate: t.number.optional(),
 
+    /** Maximum concurrent buckets across V3 chunk-compaction jobs. Range: 1–64. Default: 4 with object storage, otherwise 2. */
+    chunk_compaction_concurrency: t.number.optional(),
+
     object_storage: S3ObjectStorageConfig.optional()
   })
 );
@@ -73,6 +76,21 @@ export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
 export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
 
 export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+export const DEFAULT_CHUNK_COMPACTION_CONCURRENCY = 2;
+export const DEFAULT_OBJECT_STORAGE_CHUNK_COMPACTION_CONCURRENCY = 4;
+
+export function normalizeChunkCompactionConcurrency(value: number | undefined, hasObjectStorage = false): number {
+  const concurrency =
+    value ??
+    (hasObjectStorage ? DEFAULT_OBJECT_STORAGE_CHUNK_COMPACTION_CONCURRENCY : DEFAULT_CHUNK_COMPACTION_CONCURRENCY);
+  if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 64) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S3201,
+      'storage.chunk_compaction_concurrency must be an integer between 1 and 64'
+    );
+  }
+  return concurrency;
+}
 
 export function normalizeClearBatchThrottleRate(value: number | undefined): number {
   const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;

package/src/utils/test-utils.ts

@@ -31,6 +31,7 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
         checksumOptions: factoryOptions.checksumOptions,
         supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
         objectStorage: factoryOptions.objectStorage,
+        chunkCompactionConcurrency: factoryOptions.chunkCompactionConcurrency,
         inlineThresholdBytes: factoryOptions.inlineThresholdBytes
       });
     },

package/test/src/chunk_compaction_config.test.ts

@@ -0,0 +1,38 @@
+import { MongoStorageConfig, normalizeChunkCompactionConcurrency } from '@module/types/types.js';
+import { describe, expect, test } from 'vitest';
+
+describe('chunk compaction concurrency configuration', () => {
+  test('defaults to two workers', () => {
+    expect(normalizeChunkCompactionConcurrency(undefined)).toBe(2);
+  });
+
+  test('defaults to four workers with object storage', () => {
+    expect(normalizeChunkCompactionConcurrency(undefined, true)).toBe(4);
+  });
+
+  test.each([false, true])('preserves an explicit override with object storage %s', (hasObjectStorage) => {
+    expect(normalizeChunkCompactionConcurrency(3, hasObjectStorage)).toBe(3);
+  });
+
+  test('decodes a configured worker count', () => {
+    const config = MongoStorageConfig.decode({
+      type: 'mongodb',
+      uri: 'mongodb://localhost:27017/powersync',
+      chunk_compaction_concurrency: 4
+    });
+    expect(normalizeChunkCompactionConcurrency(config.chunk_compaction_concurrency)).toBe(4);
+  });
+
+  test.each([1, 64])('accepts boundary worker count %s', (value) => {
+    expect(normalizeChunkCompactionConcurrency(value)).toBe(value);
+  });
+
+  test.each([0, -1, 1.5, 65, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1])(
+    'rejects invalid worker count %s',
+    (value) => {
+      expect(() => normalizeChunkCompactionConcurrency(value)).toThrow(
+        'storage.chunk_compaction_concurrency must be an integer between 1 and 64'
+      );
+    }
+  );
+});

package/test/src/object_storage_usage.test.ts

@@ -123,14 +123,18 @@ describe('ObjectStorageUsage', () => {
       await expect(
         withTransaction(db, async (session) => {
           await bucketData.insertOne(document, { session });
-          await usage.applyDelta(definitionId, 123n, session);
+          const writes = db.createWriteBatch(session, { ordered: false });
+          usage.applyDelta(definitionId, 123n, writes);
+          await writes.execute();
           throw new Error('abort this attempt');
         })
       ).rejects.toThrow('abort this attempt');
 
       await withTransaction(db, async (session) => {
         await bucketData.insertOne(document, { session });
-        await usage.applyDelta(definitionId, 123n, session);
+        const writes = db.createWriteBatch(session, { ordered: false });
+        usage.applyDelta(definitionId, 123n, writes);
+        await writes.execute();
       });
 
       expect(await bucketData.countDocuments({ storage_ref: { $exists: true } })).toBe(1);
@@ -182,7 +186,11 @@ describe('ObjectStorageUsage', () => {
       const writerId = 'reused-writer';
       const usage = new ObjectStorageUsage(db, replicationStreamId, writerId);
 
-      await withTransaction(db, (session) => usage.applyDelta(definitionId, 10n, session));
+      await withTransaction(db, async (session) => {
+        const writes = db.createWriteBatch(session, { ordered: false });
+        usage.applyDelta(definitionId, 10n, writes);
+        await writes.execute();
+      });
       await db.objectStorageUsage.updateOne(
         { _id: { g: replicationStreamId, w: writerId } },
         { $set: { updated_at: new Date(0) } }
@@ -191,7 +199,11 @@ describe('ObjectStorageUsage', () => {
 
       // The folder committed first and deleted the writer shard. The writer's
       // next transaction must recreate it through the upsert.
-      await withTransaction(db, (session) => usage.applyDelta(definitionId, 4n, session));
+      await withTransaction(db, async (session) => {
+        const writes = db.createWriteBatch(session, { ordered: false });
+        usage.applyDelta(definitionId, 4n, writes);
+        await writes.execute();
+      });
       expect((await readUsageEntries(db, replicationStreamId))[0].active_bytes).toBe(14n);
 
       await db.objectStorageUsage.updateOne(