@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts

@@ -0,0 +1,229 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { BucketDefinitionId } from '@powersync/service-sync-rules';
+import { randomUUID } from 'node:crypto';
+import { MongoWriteBatch } from '../../MongoWriteBatch.js';
+import { BucketDataDocumentV3, ObjectStorageUsageDocument } from '../models.js';
+import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
+
+export const OBJECT_STORAGE_USAGE_BASE_WRITER_ID = '__base__';
+export const DEFAULT_OBJECT_STORAGE_USAGE_STALE_WRITER_MS = 30 * 60 * 1000;
+export const DEFAULT_OBJECT_STORAGE_USAGE_FOLD_LIMIT = 100;
+
+export interface ReplicationStreamObjectStorageDefinitionUsageResult {
+  replication_stream_id: number;
+  definition_id: BucketDefinitionId;
+  active_bytes: bigint;
+}
+
+export function createObjectStorageUsageWriterId(): string {
+  return randomUUID();
+}
+
+/**
+ * Tracks active S3 references. Values are signed writer-local deltas, so reference changes and
+ * accounting updates can be committed together without making replication and compaction share a
+ * hot counter document.
+ */
+export class ObjectStorageUsage {
+  constructor(
+    private readonly db: VersionedPowerSyncMongoV3,
+    private readonly replicationStreamId: number,
+    readonly writerId?: string
+  ) {
+    if (writerId === OBJECT_STORAGE_USAGE_BASE_WRITER_ID) {
+      throw new ReplicationAssertionError('The reserved object-storage usage writer id cannot be used by a writer');
+    }
+  }
+
+  static bytes(document: Pick<BucketDataDocumentV3, 'storage_ref'>): bigint {
+    return BigInt(document.storage_ref?.file_size ?? 0);
+  }
+
+  static async readAllDefinitionUsage(
+    db: VersionedPowerSyncMongoV3
+  ): Promise<ReplicationStreamObjectStorageDefinitionUsageResult[]> {
+    return db.client.withSession({ snapshot: true }, async (session) => {
+      // Deliberately read the whole usage collection. The number of replication streams is
+      // expected to stay low, so a collection scan is cheaper than issuing one _id range query
+      // per stream (and the usage collection is bounded by streams and writers, not buckets).
+      const entries = await db.objectStorageUsage
+        .aggregate<{
+          _id: { replication_stream_id: number; definition_id: BucketDefinitionId };
+          active_bytes: bigint;
+        }>(
+          [
+            { $project: { replication_stream_id: '$_id.g', definitions: { $objectToArray: '$definitions' } } },
+            { $unwind: '$definitions' },
+            {
+              $group: {
+                _id: {
+                  replication_stream_id: '$replication_stream_id',
+                  definition_id: '$definitions.k'
+                },
+                active_bytes: { $sum: '$definitions.v' }
+              }
+            }
+          ],
+          { session, readConcern: 'snapshot' }
+        )
+        .toArray()
+        .catch((error) => {
+          if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
+            return [];
+          }
+          throw error;
+        });
+
+      return entries.map((entry) => {
+        const activeBytes = BigInt(entry.active_bytes ?? 0);
+        if (activeBytes < 0n) {
+          return {
+            replication_stream_id: entry._id.replication_stream_id,
+            definition_id: entry._id.definition_id,
+            active_bytes: 0n
+          };
+        }
+        return {
+          replication_stream_id: entry._id.replication_stream_id,
+          definition_id: entry._id.definition_id,
+          active_bytes: activeBytes
+        };
+      });
+    });
+  }
+
+  applyDelta(definitionId: BucketDefinitionId, delta: bigint, writes: MongoWriteBatch): void {
+    this.applyDeltas(new Map([[definitionId, delta]]), writes);
+  }
+
+  applyDeltas(deltas: ReadonlyMap<BucketDefinitionId, bigint>, writes: MongoWriteBatch): void {
+    if (this.writerId == null) {
+      throw new ReplicationAssertionError('A writer id is required to apply object-storage usage');
+    }
+
+    const increments: Record<string, bigint> = {};
+    for (const [definitionId, delta] of deltas) {
+      if (delta === 0n) {
+        continue;
+      }
+      increments[this.definitionPath(definitionId)] = delta;
+    }
+    if (Object.keys(increments).length === 0) {
+      return;
+    }
+
+    writes.updateOne(
+      this.db.objectStorageUsage,
+      { _id: this.documentId() },
+      { $inc: increments, $currentDate: { updated_at: true } },
+      { upsert: true }
+    );
+  }
+
+  async removeDefinition(definitionId: BucketDefinitionId, session: mongo.ClientSession): Promise<void> {
+    await this.db.objectStorageUsage.updateMany(
+      { '_id.g': this.replicationStreamId },
+      { $unset: { [this.definitionPath(definitionId)]: 1 } },
+      { session }
+    );
+  }
+
+  async removeStream(session: mongo.ClientSession): Promise<void> {
+    await this.db.objectStorageUsage.deleteMany({ '_id.g': this.replicationStreamId }, { session });
+  }
+
+  async foldStaleWriterDeltas(
+    options: {
+      staleWriterMs?: number;
+      limit?: number;
+    } = {}
+  ): Promise<void> {
+    const staleWriterMs = options.staleWriterMs ?? DEFAULT_OBJECT_STORAGE_USAGE_STALE_WRITER_MS;
+    const limit = options.limit ?? DEFAULT_OBJECT_STORAGE_USAGE_FOLD_LIMIT;
+    if (limit <= 0) {
+      return;
+    }
+
+    await this.db.client.withSession((session) =>
+      session.withTransaction(
+        async () => {
+          const staleDocuments = await this.db.objectStorageUsage
+            .find(
+              {
+                '_id.g': this.replicationStreamId,
+                '_id.w': { $ne: OBJECT_STORAGE_USAGE_BASE_WRITER_ID },
+                $expr: {
+                  $lt: [
+                    '$updated_at',
+                    { $dateSubtract: { startDate: '$$NOW', unit: 'millisecond', amount: staleWriterMs } }
+                  ]
+                }
+              },
+              { session, sort: { updated_at: 1 }, limit }
+            )
+            .toArray();
+          if (staleDocuments.length === 0) {
+            return;
+          }
+
+          const deltas = new Map<BucketDefinitionId, bigint>();
+          for (const document of staleDocuments) {
+            for (const [definitionId, delta] of Object.entries(document.definitions ?? {})) {
+              this.validateDefinitionId(definitionId);
+              deltas.set(definitionId, (deltas.get(definitionId) ?? 0n) + BigInt(delta));
+            }
+          }
+
+          const increments: Record<string, bigint> = {};
+          for (const [definitionId, delta] of deltas) {
+            if (delta !== 0n) {
+              increments[this.definitionPath(definitionId)] = delta;
+            }
+          }
+          if (Object.keys(increments).length > 0) {
+            await this.db.objectStorageUsage.updateOne(
+              { _id: this.documentId(OBJECT_STORAGE_USAGE_BASE_WRITER_ID) },
+              {
+                $inc: increments,
+                $currentDate: { updated_at: true }
+              },
+              { upsert: true, session }
+            );
+          }
+
+          await this.db.objectStorageUsage.deleteMany(
+            {
+              '_id.g': this.replicationStreamId,
+              _id: { $in: staleDocuments.map((document) => document._id) }
+            },
+            { session }
+          );
+        },
+        { readConcern: { level: 'snapshot' }, writeConcern: { w: 'majority' } }
+      )
+    );
+  }
+
+  private definitionPath(definitionId: BucketDefinitionId): string {
+    this.validateDefinitionId(definitionId);
+    return `definitions.${definitionId}`;
+  }
+
+  private documentId(writerId = this.writerId): ObjectStorageUsageDocument['_id'] {
+    if (writerId == null) {
+      throw new ReplicationAssertionError('A writer id is required to identify object-storage usage');
+    }
+    return {
+      g: this.replicationStreamId,
+      w: writerId
+    };
+  }
+
+  private validateDefinitionId(definitionId: BucketDefinitionId): void {
+    if (definitionId.length === 0 || definitionId.includes('.') || definitionId.includes('$')) {
+      throw new ReplicationAssertionError(`Invalid bucket definition id for object-storage usage: ${definitionId}`);
+    }
+  }
+}

package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts

@@ -6,20 +6,174 @@ import {
   PutObjectCommand,
   S3Client
 } from '@aws-sdk/client-s3';
+import { logger } from '@powersync/lib-services-framework';
 import { acquireSemaphoreAbortable, isAbortError } from '@powersync/service-core';
+import { loadConfigsForDefaultMode, type DefaultsMode, type ResolvedDefaultsMode } from '@smithy/core/client';
 import { isThrottlingError, isTransientError } from '@smithy/core/retry';
+import { NodeHttpHandler } from '@smithy/node-http-handler';
 import { Semaphore, SemaphoreInterface } from 'async-mutex';
-import { ObjectStorageError, type ObjectStorage, type ObjectStoragePutMetadata } from './ObjectStorage.js';
+import {
+  ObjectStorageError,
+  type ObjectStorage,
+  type ObjectStorageOperationOptions,
+  type ObjectStoragePutMetadata
+} from './ObjectStorage.js';
+
+// Never statically import (or re-export) this module!
+// We want to avoid loading the AWS SDK unless the user has configured S3 object storage, since it's a large dependency.
 
-const DEFAULT_S3_OPERATION_CONCURRENCY = 16;
+const DEFAULT_S3_OPERATION_CONCURRENCY = 64;
 const S3_DELETE_PREFIX_BATCH_SIZE = 1000;
 /**
- * Slightly smaller than DEFAULT_S3_OPERATION_CONCURRENCY.
+ * Keep prefix deletion below the default shared limit to leave capacity for other operations.
  */
 const S3_DELETE_PREFIX_CONCURRENCY = 12;
 const MAX_S3_PREFIX_BYTES = 256;
 const SAFE_S3_KEY_BYTES = 896;
 
+/**
+ * Timeouts are derived from the AWS defaults mode, rather than being individually configurable.
+ *
+ * The mode - AWS_DEFAULTS_MODE or the defaults_mode storage option - describes the expected latency
+ * between this service and the object storage endpoint. We use the connection timeout it defines
+ * as the baseline for the other timeouts:
+ *
+ * | mode                   | connection | request | operation | queue |
+ * | ---------------------- | ---------- | ------- | --------- | ----- |
+ * | in-region              |       1.1s |    5.5s |       11s |   44s |
+ * | standard, cross-region |       3.1s |   15.5s |       31s |  124s |
+ * | mobile                 |        30s |    150s |      300s | 1200s |
+ *
+ * The multipliers are sized for the objects we store: bucket data chunks target 1MB, and are
+ * bounded by the 16MB BSON document limit. Even at the upper bound, the request timeout only has to
+ * cover a transfer of a few MB/s, and the operation timeout leaves room for a second attempt.
+ *
+ * `legacy`, the mode used when nothing is configured, does not define any timeouts, and is treated
+ * as `standard` here. Running without timeouts is not an option: a stalled request holds one of the
+ * limited operation slots until the process restarts, and enough of those block all reads.
+ *
+ * `auto` is also treated as `standard` - see resolveConfiguredDefaultsMode() below.
+ */
+const BASELINE_CONNECTION_TIMEOUT_MS = 3_100;
+const REQUEST_TIMEOUT_FACTOR = 5;
+const OPERATION_TIMEOUT_FACTOR = 2;
+/**
+ * Don't keep idle http sockets around for longer than this timeout.
+ *
+ * This covers the case where we have large bursts of activity during replication, followed by
+ * long periods of inactivity.
+ */
+const SOCKET_IDLE_TIMEOUT_MS = 60_000;
+
+/**
+ * How long an operation may wait for a concurrency slot, as a multiple of the operation timeout.
+ *
+ * Every slot holder now releases within the operation timeout, so the queue always drains: this is
+ * a backpressure limit rather than a deadlock guard. Waiting this long means at least
+ * `factor * concurrencyLimit` operations ahead of us each took their full deadline - 256 at the
+ * default concurrency. Real operations complete in a fraction of the deadline, so the number of
+ * queued operations this actually tolerates is far higher, well above the ~2000 uploads that the
+ * largest possible replication flush (MAX_TRANSACTION_DOC_COUNT) can enqueue at once.
+ */
+const QUEUE_TIMEOUT_FACTOR = 4;
+
+export interface S3TimeoutProfile {
+  /**
+   * Per attempt: establishing the connection, including waiting for a socket from the pool.
+   */
+  connectionTimeoutMs: number;
+  /**
+   * Per attempt: from starting the request until the response headers are received. For uploads
+   * this includes sending the body.
+   */
+  requestTimeoutMs: number;
+  /**
+   * The complete operation, including SDK retries and streaming the response body.
+   *
+   * The SDK timeouts above stop applying once the response headers are received, so this is what
+   * bounds how long a single operation can occupy a concurrency slot.
+   */
+  operationTimeoutMs: number;
+  /**
+   * Waiting for a concurrency slot, before the operation itself starts.
+   */
+  queueTimeoutMs: number;
+}
+
+export function resolveTimeoutProfile(mode: ResolvedDefaultsMode): S3TimeoutProfile {
+  // Only connectionTimeout is currently vended by the SDK, but requestTimeout is part of the same
+  // contract - use it as the hint if a future version starts defining it.
+  const { connectionTimeout, requestTimeout } = loadConfigsForDefaultMode(mode);
+  const connectionTimeoutMs = connectionTimeout ?? BASELINE_CONNECTION_TIMEOUT_MS;
+  const requestTimeoutMs = requestTimeout ?? connectionTimeoutMs * REQUEST_TIMEOUT_FACTOR;
+  const operationTimeoutMs = requestTimeoutMs * OPERATION_TIMEOUT_FACTOR;
+  return {
+    connectionTimeoutMs,
+    requestTimeoutMs,
+    operationTimeoutMs,
+    queueTimeoutMs: operationTimeoutMs * QUEUE_TIMEOUT_FACTOR
+  };
+}
+
+const AWS_DEFAULTS_MODE_ENV = 'AWS_DEFAULTS_MODE';
+const RESOLVED_DEFAULTS_MODES: ResolvedDefaultsMode[] = ['standard', 'in-region', 'cross-region', 'mobile', 'legacy'];
+
+/**
+ * Resolve the configured AWS defaults mode: the explicit option, then the AWS_DEFAULTS_MODE
+ * environment variable, then `legacy`, which is also the AWS default.
+ *
+ * Unlike the SDK's own lookup, this does not read defaults_mode from the AWS shared config file.
+ * That lookup is asynchronous, and the mode is only used to pick timeouts, which we want available
+ * synchronously so that operations never have to wait on configuration.
+ *
+ * `auto` is substituted with `standard`. It asks the SDK to detect whether it is running in the
+ * same region as the endpoint, which means querying the EC2 instance metadata service on startup:
+ *
+ * - The detection only succeeds on EC2-family compute with instance metadata reachable. Anywhere
+ *   else - including EC2 with IMDS disabled or behind a low hop limit - it silently falls back to
+ *   `standard`, so the timeouts differ by deployment with no indication of why.
+ * - It requires a resolvable region, and fails the client outright when there is none.
+ * - Timeouts that depend on where the process happens to run are hard to reason about when
+ *   diagnosing a stall.
+ *
+ * Deployments that want the tighter in-region timeouts should ask for them explicitly.
+ */
+function resolveConfiguredDefaultsMode(configured?: DefaultsMode): ResolvedDefaultsMode {
+  // An empty environment variable is treated as unset, matching the SDK.
+  const mode = (configured || process.env[AWS_DEFAULTS_MODE_ENV] || 'legacy').toLowerCase();
+  if (mode === 'auto') {
+    logger.warn(
+      `Ignoring AWS defaults mode "auto" for object storage, using "standard" instead. ` +
+        `Configure storage.object_storage.defaults_mode to select a mode explicitly.`
+    );
+    return 'standard';
+  }
+  const resolved = RESOLVED_DEFAULTS_MODES.find((candidate) => candidate === mode);
+  if (resolved == null) {
+    throw new Error(
+      `Invalid AWS defaults mode ${JSON.stringify(mode)}, expected one of ${RESOLVED_DEFAULTS_MODES.join(', ')}, auto`
+    );
+  }
+  return resolved;
+}
+
+/**
+ * An S3 operation holding a concurrency slot, released when the operation is disposed.
+ */
+interface S3Operation extends AsyncDisposable {
+  /**
+   * The caller's signal, combined with the operation deadline.
+   *
+   * Use this for the request itself, so that neither a stalled request nor a stalled response body
+   * can hold on to the slot indefinitely.
+   */
+  signal: AbortSignal;
+  /**
+   * The error to report for a failed operation.
+   */
+  error(cause: unknown): Error;
+}
+
 export interface S3ObjectStorageOptions {
   bucket: string;
   region?: string;
@@ -29,6 +183,12 @@ export interface S3ObjectStorageOptions {
   accessKeyId?: string;
   secretAccessKey?: string;
   concurrencyLimit?: number;
+  /**
+   * AWS defaults mode, used as the baseline for the request timeouts.
+   *
+   * Defaults to the AWS_DEFAULTS_MODE environment variable. `auto` is treated as `standard`.
+   */
+  defaultsMode?: DefaultsMode;
 }
 
 export class S3ObjectStorage implements ObjectStorage {
@@ -39,6 +199,7 @@ export class S3ObjectStorage implements ObjectStorage {
   private bucket: string;
   private prefix: string;
   private readonly operationSemaphore: SemaphoreInterface;
+  private readonly timeouts: S3TimeoutProfile;
 
   constructor(options: S3ObjectStorageOptions) {
     const concurrencyLimit = options.concurrencyLimit ?? DEFAULT_S3_OPERATION_CONCURRENCY;
@@ -60,10 +221,31 @@ export class S3ObjectStorage implements ObjectStorage {
     this.bucket = options.bucket;
     this.prefix = prefix;
     this.operationSemaphore = new Semaphore(concurrencyLimit);
+
+    // Resolve the mode once, and share it with the client so that its own defaults agree with ours.
+    const defaultsMode = resolveConfiguredDefaultsMode(options.defaultsMode);
+    this.timeouts = resolveTimeoutProfile(defaultsMode);
+
     this.client = new S3Client({
       region: options.region,
       endpoint: options.endpoint,
       forcePathStyle: options.forcePathStyle,
+      defaultsMode,
+      requestHandler: new NodeHttpHandler({
+        // Match the shared request limit instead of queuing again at the HTTP pool.
+        httpAgent: {
+          keepAlive: true,
+          maxSockets: concurrencyLimit,
+          // The timeout here is a little more general than an "idle timeout", but it does cover the
+          // case of closing idle sockets after a period of inactivity.
+          timeout: SOCKET_IDLE_TIMEOUT_MS
+        },
+        httpsAgent: { keepAlive: true, maxSockets: concurrencyLimit, timeout: SOCKET_IDLE_TIMEOUT_MS },
+        connectionTimeout: this.timeouts.connectionTimeoutMs,
+        requestTimeout: this.timeouts.requestTimeoutMs,
+        // Without this, exceeding requestTimeout only logs a warning.
+        throwOnRequestTimeout: true
+      }),
       credentials:
         options.accessKeyId && options.secretAccessKey
           ? { accessKeyId: options.accessKeyId, secretAccessKey: options.secretAccessKey }
@@ -71,9 +253,14 @@ export class S3ObjectStorage implements ObjectStorage {
     });
   }
 
-  async put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void> {
+  async put(
+    path: string,
+    data: Uint8Array,
+    metadata: ObjectStoragePutMetadata,
+    options?: ObjectStorageOperationOptions
+  ): Promise<void> {
     const fullPath = this.fullPath(path);
-    await using _ = await this.withOperation();
+    await using operation = await this.withOperation('upload', fullPath, options?.signal);
     try {
       await this.client.send(
         new PutObjectCommand({
@@ -82,20 +269,22 @@ export class S3ObjectStorage implements ObjectStorage {
           Body: data,
           ContentType: metadata.contentType,
           ContentEncoding: metadata.contentEncoding ?? undefined
-        })
+        }),
+        { abortSignal: operation.signal }
       );
     } catch (error) {
-      throw s3OperationError('upload', fullPath, error);
+      throw operation.error(error);
     }
   }
 
   async get(
     path: string,
-    options?: { signal?: AbortSignal }
+    options?: ObjectStorageOperationOptions
   ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }> {
     const fullPath = this.fullPath(path);
-    const signal = options?.signal;
-    await using _ = await this.withOperation(signal);
+    await using operation = await this.withOperation('download', fullPath, options?.signal);
+    // Includes the operation deadline, which also covers streaming the response body below.
+    const signal = operation.signal;
     try {
       const response = await this.client.send(
         new GetObjectCommand({
@@ -116,7 +305,7 @@ export class S3ObjectStorage implements ObjectStorage {
       let offset = 0;
       const stream = response.Body as AsyncIterable<Uint8Array>;
       for await (const chunk of stream) {
-        signal?.throwIfAborted();
+        signal.throwIfAborted();
         const nextOffset = offset + chunk.byteLength;
         if (nextOffset > contentLength) {
           throw new Error(
@@ -143,11 +332,11 @@ export class S3ObjectStorage implements ObjectStorage {
       if (err.name === 'NoSuchKey' || err.Code === 'NoSuchKey') {
         throw new ObjectStorageError(`S3 object not found: ${fullPath}`, { cause: err, retryable: false });
       }
-      throw s3OperationError('download', fullPath, err);
+      throw operation.error(err);
     }
   }
 
-  async *list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string> {
+  async *list(prefix: string, options?: ObjectStorageOperationOptions): AsyncIterable<string> {
     const fullPrefix = this.fullPath(prefix);
     let continuationToken: string | undefined;
     const signal = options?.signal;
@@ -168,12 +357,12 @@ export class S3ObjectStorage implements ObjectStorage {
     } while (continuationToken != null);
   }
 
-  async delete(paths: string[]): Promise<void> {
+  async delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void> {
     const fullPaths = paths.map((path) => ({ Key: this.fullPath(path) }));
-    await this.deleteFullPaths(fullPaths);
+    await this.deleteFullPaths(fullPaths, options?.signal);
   }
 
-  async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
+  async deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{ objectCount: number }> {
     const fullPrefix = this.fullPath(prefix);
     const signal = options?.signal;
     let continuationToken: string | undefined;
@@ -238,8 +427,10 @@ export class S3ObjectStorage implements ObjectStorage {
     continuationToken: string | undefined,
     signal?: AbortSignal
   ): Promise<ListObjectsV2CommandOutput> {
+    // Acquired outside the try, matching the other operations: withOperation() already reports its
+    // own failures as ObjectStorageError.
+    await using operation = await this.withOperation('list', fullPrefix, signal);
     try {
-      await using _ = await this.withOperation(signal);
       return await this.client.send(
         new ListObjectsV2Command({
           Bucket: this.bucket,
@@ -247,16 +438,16 @@ export class S3ObjectStorage implements ObjectStorage {
           ContinuationToken: continuationToken,
           MaxKeys: S3_DELETE_PREFIX_BATCH_SIZE
         }),
-        { abortSignal: signal }
+        { abortSignal: operation.signal }
       );
     } catch (error) {
-      throw s3OperationError('list', fullPrefix, error);
+      throw operation.error(error);
     }
   }
 
   private async deleteFullPaths(fullPaths: { Key: string }[], signal?: AbortSignal): Promise<void> {
     if (fullPaths.length === 0) return;
-    await using _ = await this.withOperation(signal);
+    await using operation = await this.withOperation('delete', `${fullPaths.length} objects`, signal);
     let response;
     try {
       response = await this.client.send(
@@ -264,10 +455,10 @@ export class S3ObjectStorage implements ObjectStorage {
           Bucket: this.bucket,
           Delete: { Objects: fullPaths, Quiet: true }
         }),
-        { abortSignal: signal }
+        { abortSignal: operation.signal }
       );
     } catch (error) {
-      throw s3OperationError('delete', `${fullPaths.length} objects`, error);
+      throw operation.error(error);
     }
     if (response.Errors?.length) {
       const errors = response.Errors.map((error) => `${error.Key}: ${error.Code}`).join(', ');
@@ -287,14 +478,28 @@ export class S3ObjectStorage implements ObjectStorage {
     return fullPath;
   }
 
-  private async withOperation(signal?: AbortSignal): Promise<AsyncDisposable> {
+  private async withOperation(operation: string, target: string, signal?: AbortSignal): Promise<S3Operation> {
     signal?.throwIfAborted();
-    const acquired = signal
-      ? await acquireSemaphoreAbortable(this.operationSemaphore, signal)
-      : await this.operationSemaphore.acquire();
+    const { operationTimeoutMs, queueTimeoutMs } = this.timeouts;
+
+    // Waiting for a slot is bounded even when the caller has no signal of its own, which is the
+    // case for uploads and deletes.
+    const queueDeadline = timeoutSignal(queueTimeoutMs, () =>
+      s3TimeoutError(operation, target, `timed out after ${queueTimeoutMs} ms waiting for a concurrency slot`)
+    );
+    let acquired;
+    try {
+      acquired = await acquireSemaphoreAbortable(
+        this.operationSemaphore,
+        signal ? AbortSignal.any([signal, queueDeadline.signal]) : queueDeadline.signal
+      );
+    } finally {
+      queueDeadline.cancel();
+    }
     if (acquired === 'aborted') {
-      signal!.throwIfAborted();
-      throw new Error('S3 operation aborted while waiting for a concurrency slot');
+      // A caller abort takes precedence: it is a cancellation, not an S3 failure.
+      signal?.throwIfAborted();
+      throw queueDeadline.signal.reason;
     }
 
     const [, release] = acquired;
@@ -302,12 +507,57 @@ export class S3ObjectStorage implements ObjectStorage {
       release();
       signal.throwIfAborted();
     }
+    // This deadline only starts once the operation holds a slot - queueing is covered above.
+    const deadline = timeoutSignal(operationTimeoutMs, () =>
+      s3TimeoutError(operation, target, `did not complete within ${operationTimeoutMs} ms`)
+    );
+    const operationSignal = signal ? AbortSignal.any([signal, deadline.signal]) : deadline.signal;
     return {
-      [Symbol.asyncDispose]: async () => release()
+      signal: operationSignal,
+      error: (cause) => {
+        // Aborting a request that is already streaming destroys the socket, and the failure
+        // surfaces as a generic "aborted" error that says nothing about the cause. The abort
+        // reason is always the better answer: the caller's reason when the caller stopped us -
+        // callers abort with the error that stopped them, and upstream handling matches on that
+        // type - and an already-classified timeout when the deadline did.
+        if (operationSignal.aborted) {
+          return operationSignal.reason as Error;
+        }
+        return s3OperationError(operation, target, cause);
+      },
+      [Symbol.asyncDispose]: async () => {
+        deadline.cancel();
+        release();
+      }
     };
   }
 }
 
+/**
+ * An abort signal that fires after the given timeout, using a timer that does not hold the process
+ * open.
+ *
+ * The reason is what callers see when the timeout fires, so it is the error to report - not a
+ * marker to translate later.
+ */
+function timeoutSignal(timeoutMs: number, createReason: () => unknown): { signal: AbortSignal; cancel: () => void } {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(createReason()), timeoutMs);
+  timer.unref();
+  return { signal: controller.signal, cancel: () => clearTimeout(timer) };
+}
+
+/**
+ * A timeout that the compactor may retry, matching how the AWS SDK classifies its own request
+ * timeouts.
+ */
+function s3TimeoutError(operation: string, target: string, message: string): ObjectStorageError {
+  return new ObjectStorageError(`S3 ${operation} for ${target} ${message}`, {
+    cause: new DOMException(message, 'TimeoutError'),
+    retryable: true
+  });
+}
+
 function s3OperationError(operation: string, target: string, error: unknown): Error {
   if (error instanceof ObjectStorageError) {
     return error;