@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

package/test/src/storage_s3_checksums.test.ts

@@ -4,7 +4,7 @@ import {
   storage,
   updateSyncRulesFromYaml
 } from '@powersync/service-core';
-import { bucketRequest, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
 import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
@@ -60,8 +60,7 @@ describe('V3 checksums with S3 object storage', () => {
       {
         $set: {
           last_op: 3n,
-          compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: null },
-          estimate_since_compact: { count: 3, bytes: 100 }
+          compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: 0n, chunks: 0, at: new Date() }
         }
       },
       { upsert: true }
@@ -154,7 +153,7 @@ describe('V3 checksums with S3 object storage', () => {
     const checkpoint = await bucketStorage.getCheckpoint();
 
     // Compact — produces CLEAR doc from collapsed MOVEs
-    await bucketStorage.compact({
+    await compactActive(factory, {
       maxOpId: checkpoint.checkpoint,
       compactBuckets: [request.bucket],
       clearBatchLimit: 200,

package/test/src/storage_s3_compaction_lifecycle.test.ts

@@ -1,7 +1,9 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
+import { MongoSyncBucketStorageV3 } from '../../src/storage/implementation/v3/MongoSyncBucketStorageV3.js';
 import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
 import { ObjectStorageError } from '../../src/storage/implementation/v3/object-storage/ObjectStorage.js';
 import { env } from './env.js';
@@ -28,8 +30,30 @@ function memoryS3Factory(options: { inlineThresholdBytes?: number } = {}) {
   return { memoryStorage: objectStorage, factory: factoryGen };
 }
 
+async function expectUsageMatchesBucketData(bucketStorage: MongoSyncBucketStorage, definitionId: string) {
+  const v3BucketStorage = bucketStorage as MongoSyncBucketStorageV3;
+  const db = v3BucketStorage.db as VersionedPowerSyncMongoV3;
+  const documents = await db.bucketData(v3BucketStorage.replicationStreamId, definitionId).find({}).toArray();
+  const expectedBytes = documents.reduce((sum, document) => sum + BigInt(document.storage_ref?.file_size ?? 0), 0n);
+  const entries = await db.objectStorageUsage
+    .aggregate<{ active_bytes: bigint }>([
+      { $match: { '_id.g': v3BucketStorage.replicationStreamId } },
+      { $project: { definitions: { $objectToArray: '$definitions' } } },
+      { $unwind: '$definitions' },
+      { $group: { _id: null, active_bytes: { $sum: '$definitions.v' } } }
+    ])
+    .toArray()
+    .catch((error) => {
+      if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
+        return [];
+      }
+      throw error;
+    });
+  expect(BigInt(entries[0]?.active_bytes ?? 0)).toBe(expectedBytes);
+}
+
 describe('S3 compaction storage lifecycle', () => {
-  test('retries transient object storage failures', async () => {
+  test('a later compaction recovers from a transient object storage failure', async () => {
     const { memoryStorage, factory: factoryGen } = memoryS3Factory();
     await using factory = await factoryGen.factory();
     const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
@@ -63,12 +87,14 @@ describe('S3 compaction storage lifecycle', () => {
 
     const checkpoint = await bucketStorage.getCheckpoint();
     const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
-    await bucketStorage.compact({
+    const compactOptions = {
       maxOpId: checkpoint.checkpoint,
       compactBuckets: [request.bucket],
       minBucketChanges: 1,
       minChangeRatio: 0
-    });
+    };
+    await expect(compactActive(factory, compactOptions)).rejects.toThrow('temporary object storage failure');
+    await compactActive(factory, compactOptions);
 
     expect(injectedFailure).toBe(true);
     const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
@@ -107,7 +133,7 @@ describe('S3 compaction storage lifecycle', () => {
 
     const checkpoint = await bucketStorage.getCheckpoint();
     const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
-    await bucketStorage.compact({
+    await compactActive(factory, {
       maxOpId: checkpoint.checkpoint,
       compactBuckets: [request.bucket],
       minBucketChanges: 1,
@@ -119,6 +145,103 @@ describe('S3 compaction storage lifecycle', () => {
     expect(docs[0].storage_ref).toBeUndefined();
     expect(docs[0].ops).toBeDefined();
     expect(memoryStorage.store.size).toBe(0);
+    await expectUsageMatchesBucketData(bucketStorage, definitionId);
+  });
+
+  test('boundary CLEAR replacement updates active object-storage usage', async () => {
+    const { factory: factoryGen } = memoryS3Factory();
+    await using factory = await factoryGen.factory();
+    const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
+    const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
+
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 11);
+    await writer.markAllSnapshotDone('1/1');
+    for (const [index, description] of ['first', 'second', 'latest'].entries()) {
+      await writer.save({
+        sourceTable,
+        tag: index === 0 ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE,
+        after: { id: 'A', description: description.repeat(100) },
+        afterReplicaId: test_utils.rid('A')
+      });
+    }
+    await writer.commit('1/1');
+
+    const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
+    const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
+    const checkpoint = await bucketStorage.getCheckpoint();
+    await compactActive(factory, {
+      maxOpId: checkpoint.checkpoint,
+      compactBuckets: [request.bucket],
+      minBucketChanges: 1,
+      minChangeRatio: 0
+    });
+
+    const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const data = batch.flatMap((chunk) => chunk.chunkData.data);
+    expect(data.some((op: any) => op.op === 'CLEAR')).toBe(true);
+    await expectUsageMatchesBucketData(bucketStorage, definitionId);
+  });
+
+  test('leading and boundary CLEAR replacements update active object-storage usage', async () => {
+    const { factory: factoryGen } = memoryS3Factory();
+    await using factory = await factoryGen.factory();
+    const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
+    const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
+
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 12);
+    await writer.markAllSnapshotDone('1/1');
+
+    // The first and last A versions force a leading MOVE document. The
+    // middle document also contains a large surviving PUT, keeping it as the
+    // CLEAR boundary document rather than merging it with the leading group.
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: { id: 'A', description: 'old' },
+      afterReplicaId: test_utils.rid('A')
+    });
+    await writer.commit('1/1');
+
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.UPDATE,
+      after: { id: 'A', description: 'middle' },
+      afterReplicaId: test_utils.rid('A')
+    });
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: { id: 'B', description: 'b'.repeat(990_000) },
+      afterReplicaId: test_utils.rid('B')
+    });
+    await writer.commit('2/1');
+
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.UPDATE,
+      after: { id: 'A', description: 'latest'.repeat(10_000) },
+      afterReplicaId: test_utils.rid('A')
+    });
+    await writer.commit('3/1');
+
+    const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
+    const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
+    const checkpoint = await bucketStorage.getCheckpoint();
+    await compactActive(factory, {
+      maxOpId: checkpoint.checkpoint,
+      compactBuckets: [request.bucket],
+      clearBatchLimit: 2,
+      moveBatchQueryLimit: 1,
+      minBucketChanges: 1,
+      minChangeRatio: 0
+    });
+
+    const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const data = batch.flatMap((chunk) => chunk.chunkData.data);
+    expect(data.some((op: any) => op.op === 'CLEAR')).toBe(true);
+    await expectUsageMatchesBucketData(bucketStorage, definitionId);
   });
 
   test('MOVE compaction merges adjacent objects using their compacted size in one write', async () => {
@@ -167,7 +290,7 @@ describe('S3 compaction storage lifecycle', () => {
 
     const checkpoint = await bucketStorage.getCheckpoint();
     const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
-    await bucketStorage.compact({
+    await compactActive(factory, {
       maxOpId: checkpoint.checkpoint,
       compactBuckets: [request.bucket],
       clearBatchLimit: 200,
@@ -193,6 +316,7 @@ describe('S3 compaction storage lifecycle', () => {
     // final replacement was uploaded. There was no intermediate MOVE-only
     // object before the merge.
     expect(objectStore.size).toBe(4);
+    await expectUsageMatchesBucketData(bucketStorage, definitionId);
   });
 
   test('compaction only retires S3 objects at or below the maxOpId horizon', async () => {
@@ -230,7 +354,7 @@ describe('S3 compaction storage lifecycle', () => {
     const lowerPaths = new Set(docsBefore.slice(0, 6).map((doc) => doc.storage_ref!.path));
     const upperPaths = new Set(docsBefore.slice(6).map((doc) => doc.storage_ref!.path));
 
-    await bucketStorage.compact({
+    await compactActive(factory, {
       maxOpId,
       compactBuckets: [request.bucket],
       clearBatchLimit: 200,
@@ -257,6 +381,7 @@ describe('S3 compaction storage lifecycle', () => {
 
     // Retired objects remain readable during the reference grace period.
     expect([...lowerPaths].every((path) => memoryStorage.store.has(path))).toBe(true);
+    await expectUsageMatchesBucketData(bucketStorage, definitionId);
 
     const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(readCheckpoint, [request]));
     const data = batch.flatMap((chunk) => chunk.chunkData.data);
@@ -337,14 +462,14 @@ describe('S3 compaction storage lifecycle', () => {
     const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
     const bucket = request.bucket;
 
-    // Read bucket_state before compaction to confirm it exists and has
-    // estimate_since_compact populated by the writer.
+    // Read bucket_state before compaction to confirm the writer recorded its
+    // aggregate statistics and scheduled a compact check.
     const bucketStateBefore = await bucketStateCollection.findOne({
       _id: { d: definitionId, b: bucket }
     });
     expect(bucketStateBefore).toBeDefined();
-    expect(bucketStateBefore!.estimate_since_compact).toBeDefined();
-    expect(bucketStateBefore!.estimate_since_compact!.count).toBeGreaterThan(0);
+    expect(bucketStateBefore!.bucket_stats.count).toBeGreaterThan(0);
+    expect(bucketStateBefore!.next_compact_check).toBeDefined();
 
     // Record the input operations and object path.
     const batchBefore = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
@@ -357,7 +482,7 @@ describe('S3 compaction storage lifecycle', () => {
     expect(expectedCompactedOpId).not.toBeNull();
 
     // Compact the bucket.
-    await bucketStorage.compact({
+    await compactActive(factory, {
       maxOpId: checkpoint.checkpoint,
       compactBuckets: [bucket],
       clearBatchLimit: 200,
@@ -402,6 +527,7 @@ describe('S3 compaction storage lifecycle', () => {
     for (const path of afterS3Paths) {
       expect(storedPaths.has(path)).toBe(true);
     }
+    await expectUsageMatchesBucketData(bucketStorage, definitionId);
 
     // The replacement object is readable and contains the expected compacted
     // operation sequence.

package/test/src/storage_s3_reading.test.ts

@@ -2,12 +2,16 @@ import { DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, test_utils } from '@powersync/service-core-tests';
 import * as bson from 'bson';
-import { describe, expect, test } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
 import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
 import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
 import { hydrateBucketDataDocuments } from '../../src/storage/implementation/v3/object-storage/BucketDataObjectStorage.js';
 import { ObjectStorageError } from '../../src/storage/implementation/v3/object-storage/ObjectStorage.js';
-import { S3ObjectStorage } from '../../src/storage/implementation/v3/object-storage/S3ObjectStorage.js';
+import {
+  resolveTimeoutProfile,
+  S3ObjectStorage,
+  type S3TimeoutProfile
+} from '../../src/storage/implementation/v3/object-storage/S3ObjectStorage.js';
 import { env } from './env.js';
 import { MemoryObjectStorage } from './helpers/MemoryObjectStorage.js';
 import { createMemoryS3TestStorageSuite, createS3TestStorageSuite } from './helpers/s3TestFactory.js';
@@ -62,6 +66,234 @@ describe('S3 object storage reads', () => {
     );
   });
 
+  test('derives request timeouts from the AWS defaults mode', () => {
+    // legacy - the mode used when nothing is configured - defines no timeouts of its own,
+    // and uses the same baseline as standard.
+    expect(resolveTimeoutProfile('legacy')).toEqual({
+      connectionTimeoutMs: 3_100,
+      requestTimeoutMs: 15_500,
+      operationTimeoutMs: 31_000,
+      queueTimeoutMs: 124_000
+    });
+    expect(resolveTimeoutProfile('standard')).toEqual(resolveTimeoutProfile('legacy'));
+    expect(resolveTimeoutProfile('cross-region')).toEqual(resolveTimeoutProfile('legacy'));
+    expect(resolveTimeoutProfile('in-region')).toEqual({
+      connectionTimeoutMs: 1_100,
+      requestTimeoutMs: 5_500,
+      operationTimeoutMs: 11_000,
+      queueTimeoutMs: 44_000
+    });
+    expect(resolveTimeoutProfile('mobile')).toEqual({
+      connectionTimeoutMs: 30_000,
+      requestTimeoutMs: 150_000,
+      operationTimeoutMs: 300_000,
+      queueTimeoutMs: 1_200_000
+    });
+  });
+
+  test('resolves explicit and environment defaults modes without waiting on configuration', async () => {
+    // The timeouts are resolved up front, so operations never wait on configuration. The client
+    // wraps the mode in a provider of its own, which resolves to the same value.
+    const modeOf = async (objectStorage: S3ObjectStorage) => {
+      const configured = objectStorage.client.config.defaultsMode;
+      return typeof configured === 'function' ? await configured() : configured;
+    };
+
+    const previousMode = process.env.AWS_DEFAULTS_MODE;
+    try {
+      process.env.AWS_DEFAULTS_MODE = 'cross-region';
+      await expect(modeOf(new S3ObjectStorage({ bucket: 'test', region: 'test' }))).resolves.toEqual('cross-region');
+
+      // The storage option takes precedence over the environment.
+      await expect(
+        modeOf(new S3ObjectStorage({ bucket: 'test', region: 'test', defaultsMode: 'in-region' }))
+      ).resolves.toEqual('in-region');
+
+      // 'auto' would query the EC2 instance metadata service, and requires a resolvable region.
+      await expect(modeOf(new S3ObjectStorage({ bucket: 'test', defaultsMode: 'auto' }))).resolves.toEqual('standard');
+
+      delete process.env.AWS_DEFAULTS_MODE;
+      await expect(modeOf(new S3ObjectStorage({ bucket: 'test', region: 'test' }))).resolves.toEqual('legacy');
+
+      // An invalid mode fails on startup, rather than on every operation.
+      expect(
+        () => new S3ObjectStorage({ bucket: 'test', region: 'test', defaultsMode: 'nonsense' as any })
+      ).toThrowError(/Invalid AWS defaults mode "nonsense"/);
+    } finally {
+      if (previousMode == null) {
+        delete process.env.AWS_DEFAULTS_MODE;
+      } else {
+        process.env.AWS_DEFAULTS_MODE = previousMode;
+      }
+    }
+  });
+
+  test('cancels active uploads and deletes', async () => {
+    const operations = [
+      (objectStorage: S3ObjectStorage, signal: AbortSignal) =>
+        objectStorage.put(
+          'object',
+          new Uint8Array(),
+          { contentType: 'application/bson', contentEncoding: null },
+          { signal }
+        ),
+      (objectStorage: S3ObjectStorage, signal: AbortSignal) => objectStorage.delete(['object'], { signal })
+    ];
+
+    for (const startOperation of operations) {
+      const objectStorage = new S3ObjectStorage({ bucket: 'test', region: 'test' });
+      const controller = new AbortController();
+      let requestStarted!: () => void;
+      const started = new Promise<void>((resolve) => (requestStarted = resolve));
+      objectStorage.client.send = async (_command: any, options: any) => {
+        requestStarted();
+        await new Promise<void>((_resolve, reject) => {
+          const abort = () => reject(options.abortSignal.reason);
+          if (options.abortSignal.aborted) {
+            abort();
+          } else {
+            options.abortSignal.addEventListener('abort', abort, { once: true });
+          }
+        });
+        throw new Error('unreachable');
+      };
+
+      const running = startOperation(objectStorage, controller.signal);
+      const expectation = expect(running).rejects.toMatchObject({ name: 'AbortError' });
+      await started;
+      controller.abort();
+      await expectation;
+    }
+  });
+
+  test('reports why a streaming operation was cancelled', async () => {
+    // Aborting mid-stream destroys the socket, and the stream fails with a generic 'aborted'
+    // error. The operation reports the actual cause instead.
+    const stalledDownload = (objectStorage: S3ObjectStorage) => {
+      objectStorage.client.send = async (_command: any, options: any) => ({
+        ContentLength: 10,
+        ContentType: 'application/bson',
+        Body: (async function* () {
+          yield new Uint8Array([1, 2, 3]);
+          await new Promise<void>((_resolve, reject) => {
+            const abort = () => reject(new Error('stream aborted'));
+            if (options.abortSignal.aborted) {
+              abort();
+            } else {
+              options.abortSignal.addEventListener('abort', abort, { once: true });
+            }
+          });
+        })()
+      });
+    };
+
+    const timingOut = new S3ObjectStorage({ bucket: 'test', region: 'test' });
+    (timingOut as any).timeouts = {
+      ...resolveTimeoutProfile('standard'),
+      operationTimeoutMs: 50
+    } satisfies S3TimeoutProfile;
+    stalledDownload(timingOut);
+    await expect(timingOut.get('stalled-object')).rejects.toMatchObject({
+      name: 'ObjectStorageError',
+      retryable: true,
+      message: expect.stringContaining('did not complete within 50 ms')
+    } satisfies Partial<ObjectStorageError>);
+
+    // A caller abort is a cancellation, not a retryable failure.
+    const cancelled = new S3ObjectStorage({ bucket: 'test', region: 'test' });
+    stalledDownload(cancelled);
+    const controller = new AbortController();
+    const reading = cancelled.get('stalled-object', { signal: controller.signal });
+    const expectation = expect(reading).rejects.toMatchObject({ name: 'AbortError' });
+    controller.abort();
+    await expectation;
+  });
+
+  test('reports the caller abort reason unchanged', async () => {
+    // Callers abort with the error that stopped them, and upstream error handling matches on that
+    // type. Replacing it with an ObjectStorageError would hide it - for MongoDB replication, that
+    // loses the invalidation error that triggers a replication restart.
+    class StreamInvalidatedError extends Error {}
+
+    const objectStorage = new S3ObjectStorage({ bucket: 'test', region: 'test' });
+    const reason = new StreamInvalidatedError('stream invalidated');
+    const controller = new AbortController();
+    objectStorage.client.send = async (_command: any, options: any) => {
+      controller.abort(reason);
+      throw options.abortSignal.reason;
+    };
+
+    await expect(objectStorage.get('object', { signal: controller.signal })).rejects.toBe(reason);
+    await expect(
+      objectStorage.put(
+        'object',
+        new Uint8Array(),
+        { contentType: 'application/bson', contentEncoding: null },
+        { signal: controller.signal }
+      )
+    ).rejects.toBe(reason);
+    await expect(objectStorage.delete(['object'], { signal: controller.signal })).rejects.toBe(reason);
+  });
+
+  test('times out waiting for an operation slot', async () => {
+    const objectStorage = new S3ObjectStorage({ bucket: 'test', region: 'test', concurrencyLimit: 1 });
+    (objectStorage as any).timeouts = {
+      connectionTimeoutMs: 10,
+      requestTimeoutMs: 20,
+      operationTimeoutMs: 10_000,
+      queueTimeoutMs: 50
+    } satisfies S3TimeoutProfile;
+    let releaseBlocker!: () => void;
+    const blocked = new Promise<void>((resolve) => (releaseBlocker = resolve));
+    objectStorage.client.send = async () => {
+      await blocked;
+      return { ContentLength: 0, Body: (async function* () {})(), ContentType: 'application/bson' };
+    };
+
+    // The single slot is held by the first upload, which has no signal of its own.
+    const holding = objectStorage.put('held-object', new Uint8Array(), {
+      contentType: 'application/bson',
+      contentEncoding: null
+    });
+    await expect(objectStorage.get('queued-object')).rejects.toMatchObject({
+      name: 'ObjectStorageError',
+      retryable: true,
+      message: expect.stringContaining('waiting for a concurrency slot')
+    } satisfies Partial<ObjectStorageError>);
+
+    releaseBlocker();
+    await holding;
+    // The slot is usable again once the blocking operation completes.
+    await expect(objectStorage.get('later-object')).resolves.toMatchObject({ data: new Uint8Array(0) });
+  });
+
+  test('reports a caller abort while queued as a cancellation', async () => {
+    const objectStorage = new S3ObjectStorage({ bucket: 'test', region: 'test', concurrencyLimit: 1 });
+    const controller = new AbortController();
+    let holderStarted!: () => void;
+    const started = new Promise<void>((resolve) => (holderStarted = resolve));
+    let releaseHolder!: () => void;
+    const blocked = new Promise<void>((resolve) => (releaseHolder = resolve));
+    objectStorage.client.send = async () => {
+      holderStarted();
+      await blocked;
+      return {};
+    };
+
+    const holding = objectStorage.put('held-object', new Uint8Array(), {
+      contentType: 'application/bson',
+      contentEncoding: null
+    });
+    await started;
+
+    const queuedRead = objectStorage.get('queued-object', { signal: controller.signal });
+    const expectation = expect(queuedRead).rejects.toMatchObject({ name: 'AbortError' });
+    controller.abort();
+    await expectation;
+    releaseHolder();
+    await holding;
+  });
+
   test('rejects complete S3 keys over the safe byte limit', async () => {
     const objectStorage = new S3ObjectStorage({
       bucket: 'test',
@@ -125,6 +357,16 @@ describe('S3 object storage reads', () => {
       message: expect.stringContaining('S3 download failed for transient-object')
     } satisfies Partial<ObjectStorageError>);
 
+    const timeoutCause = new DOMException('The operation was aborted due to timeout', 'TimeoutError');
+    objectStorage.client.send = async () => {
+      throw timeoutCause;
+    };
+    await expect(objectStorage.get('timed-out-object')).rejects.toMatchObject({
+      name: 'ObjectStorageError',
+      retryable: true,
+      cause: timeoutCause
+    } satisfies Partial<ObjectStorageError>);
+
     const permanentCause = Object.assign(new Error('access denied'), {
       name: 'AccessDenied',
       $metadata: { httpStatusCode: 403 }
@@ -334,6 +576,52 @@ describe('S3 object storage reads', () => {
     expect(maxActiveOperations).toBe(4);
   });
 
+  test('drains sibling downloads before reporting a hydration failure', async () => {
+    const objectStorage = new MemoryObjectStorage();
+    const gate = Promise.withResolvers<void>();
+    const failure = new Error('download failed');
+    const documents = ['failed', 'pending'].map((path, index) => ({
+      _id: { b: 'bucket', o: BigInt(index + 1) },
+      min_op: BigInt(index + 1),
+      checksum: 0n,
+      count: 0,
+      size: 1,
+      storage_ref: { path, file_size: 1 }
+    }));
+    const get = vi.spyOn(objectStorage, 'get').mockImplementation(async (path) => {
+      if (path === 'failed') {
+        throw failure;
+      }
+      await gate.promise;
+      return {
+        data: bson.serialize({ ops: [] }),
+        metadata: { contentType: 'application/bson', contentEncoding: null }
+      };
+    });
+    let settled = false;
+    const result = hydrateBucketDataDocuments(documents, objectStorage, {}).then(
+      () => {
+        settled = true;
+        return undefined;
+      },
+      (error) => {
+        settled = true;
+        return error;
+      }
+    );
+    try {
+      await vi.waitFor(() => expect(get).toHaveBeenCalledTimes(2));
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      expect(settled).toBe(false);
+    } finally {
+      gate.resolve();
+      await result;
+      get.mockRestore();
+    }
+    expect(await result).toBe(failure);
+    expect(documents[1]).toHaveProperty('ops', []);
+  });
+
   test('aborts active object downloads', async () => {
     const objectStorage = new MemoryObjectStorage();
     const controller = new AbortController();

package/test/src/storage_s3_writing.test.ts

@@ -65,6 +65,7 @@ describe('S3 object storage writes', () => {
       { op: 'PUT', row_id: 'item2' }
     ]);
     expect(ops.every((op) => op.data != null)).toBe(true);
+    expect(ops.every((op) => typeof op.subkey == 'string')).toBe(true);
 
     // Verify the corresponding MongoDB metadata shell.
     const db = bucketStorage.db as VersionedPowerSyncMongoV3;