@platformatic/sqlite-js-stream-s3 0.2.1 → 0.4.0

package/src/replica-client.ts

@@ -10,6 +10,26 @@
  *
  * Creation timestamps are preserved on upload as the `ltx-timestamp` object
  * metadata; listing falls back to the object's `LastModified`.
+ *
+ * The client also supports *fenced writes* (see {@link S3ReplicaClient.registerGeneration}).
+ * S3 cannot check one object and write another in a single request, so fencing
+ * is a protocol built from conditional writes:
+ *
+ * - `{prefix}/generation.json` holds the current generation and a list of
+ *   *fences*. A fence `(generation, txid)` records the latest TXID of the
+ *   replica right after `generation` was registered. Registering is a
+ *   conditional write (`If-Match` / `If-None-Match: *`) that only raises the
+ *   generation; the fence is then added with another conditional write.
+ * - LTX files of a fenced client are written with `If-None-Match: *`, so a TXID
+ *   range is written once per level, and carry their generation in the
+ *   `ltx-generation` metadata. A file with no `ltx-generation` is generation 0.
+ * - Listings ignore a file whose generation is lower than a fence's generation
+ *   and whose `maxTXID` is above that fence's TXID: it was written by a
+ *   superseded writer after the new one read the replica.
+ * - After each upload, a fenced client re-reads the generation object and fails
+ *   with {@link ErrGenerationSuperseded} if a higher generation is registered.
+ *   An upload that resolves was therefore written while its generation was the
+ *   latest, and every later generation sees it.
  */
 
 import { Readable } from 'node:stream';
@@ -29,8 +49,11 @@ import {
 } from '@aws-sdk/client-s3';
 
 import {
+  DefaultLeasePath,
+  ErrGenerationSuperseded,
   HeaderSize,
   LTXError,
+  SnapshotLevel,
   formatFilename,
   noopLogger,
   parseFilename,
@@ -43,6 +66,8 @@ import {
   type WriteLTXFileOptions,
 } from '@platformatic/sqlite-js-stream';
 
+import { isConditionalConflict, isNotFound, isPreconditionFailed, readBody } from './util.ts';
+
 /** The strategy identifier reported by {@link S3ReplicaClient.type}. */
 export const ReplicaClientType = 's3';
 
@@ -69,8 +94,171 @@ export interface S3ReplicaClientOptions {
   sessionToken?: string;
   /** Inject a preconfigured client. Mostly for tests. */
   client?: S3ClientLike;
+  /**
+   * Fenced-write generation, or `null` (the default) to disable fencing. Set
+   * by {@link S3ReplicaClient.registerGeneration}; when set, uploads and
+   * deletions fail with {@link ErrGenerationSuperseded} once a higher
+   * generation is registered.
+   */
+  generation?: bigint | null;
+}
+
+/** Configuration for {@link S3ReplicaClient.deletePrefix}. */
+export interface S3DeletePrefixOptions {
+  /** S3 bucket name. Required. */
+  bucket: string;
+  /**
+   * Prefix to delete: the replica at this prefix and every replica under
+   * `prefix/` (`a` covers `a` and `a/b`, not `ab`). Leading/trailing slashes
+   * are normalized away. Must not be empty.
+   */
+  prefix: string;
+  /** AWS region. Defaults to the AWS SDK region resolution chain. */
+  region?: string;
+  /** Custom endpoint URL (e.g. for MinIO / LocalStack). */
+  endpoint?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  sessionToken?: string;
+  /** Inject a preconfigured client. */
+  client?: S3ClientLike;
+  /**
+   * Fences the deletion: it fails with {@link ErrGenerationSuperseded} if a
+   * replica under the prefix has a higher generation, checked before deleting
+   * any of its objects (best effort, see {@link S3ReplicaClient.deletePrefix}).
+   * `null` (the default) deletes unconditionally.
+   */
+  generation?: bigint | null;
+}
+
+/** Name of the generation object, relative to the replica prefix. */
+export const GenerationObjectName = 'generation.json';
+
+/** Object metadata key carrying the generation of a fenced LTX upload. */
+export const GenerationMetadataKey = 'ltx-generation';
+
+/**
+ * A fence: `generation` was registered when the latest TXID of the replica was
+ * `txid`. Files of lower generations with a `maxTXID` above `txid` are ignored.
+ */
+export interface GenerationFence {
+  generation: bigint;
+  txid: bigint;
+}
+
+/** Contents of the generation object. */
+export interface GenerationState {
+  /** The highest registered generation. */
+  generation: bigint;
+  /** Fences, in registration order (one per generation that completed registration). */
+  fences: GenerationFence[];
+}
+
+/**
+ * Returns true if a file of `generation` ending at `maxTXID` is part of the
+ * replica: no fence of a higher generation was recorded below `maxTXID`.
+ */
+export function isFenced(fences: readonly GenerationFence[], generation: bigint, maxTXID: bigint): boolean {
+  for (const f of fences) {
+    if (f.generation > generation && maxTXID > f.txid) return false;
+  }
+  return true;
+}
+
+/** Matches the key of an LTX file, capturing the prefix of its replica. */
+const LTX_KEY_RE = /^(.+)\/ltx\/\d+\/[^/]+$/;
+
+/** Returns the prefix of the replica a key belongs to, or null if none. */
+function replicaOfKey(key: string): string | null {
+  for (const name of [GenerationObjectName, DefaultLeasePath]) {
+    if (key.endsWith(`/${name}`)) return key.slice(0, -(name.length + 1));
+  }
+  return LTX_KEY_RE.exec(key)?.[1] ?? null;
+}
+
+/** Builds an S3 client from connection options, unless one is injected. */
+function createClient(opts: {
+  client?: S3ClientLike;
+  region?: string;
+  endpoint?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  sessionToken?: string;
+}): S3ClientLike {
+  return (
+    opts.client ??
+    new S3Client({
+      region: opts.region,
+      endpoint: opts.endpoint,
+      credentials: opts.accessKeyId
+        ? {
+            accessKeyId: opts.accessKeyId,
+            secretAccessKey: opts.secretAccessKey ?? '',
+            sessionToken: opts.sessionToken,
+          }
+        : undefined,
+    })
+  );
+}
+
+/** Retries of a conditional write that keeps conflicting with concurrent ones. */
+const MAX_CONDITIONAL_ATTEMPTS = 10;
+
+/** HeadObject requests in flight while resolving the generation of listed files. */
+const HEAD_CONCURRENCY = 16;
+
+/** Cached generations of listed objects (by key and ETag). */
+const GENERATION_CACHE_SIZE = 10_000;
+
+/** Parses the `ltx-generation` metadata (0 when absent or invalid). */
+function metadataGeneration(metadata: Record<string, string> | undefined): bigint {
+  const v = metadata?.[GenerationMetadataKey];
+  if (v === undefined || !/^\d+$/.test(v)) return 0n;
+  return BigInt(v);
+}
+
+/** Serializes a generation state (bigints as decimal strings). */
+function encodeGenerationState(state: GenerationState): Uint8Array {
+  return new TextEncoder().encode(
+    JSON.stringify({
+      generation: state.generation.toString(),
+      fences: state.fences.map((f) => ({ generation: f.generation.toString(), txid: f.txid.toString() })),
+    }),
+  );
+}
+
+/** Parses a generation object body. */
+function decodeGenerationState(data: Uint8Array): GenerationState {
+  const parsed = JSON.parse(new TextDecoder().decode(data)) as {
+    generation?: string;
+    fences?: Array<{ generation?: string; txid?: string }>;
+  };
+  return {
+    generation: BigInt(parsed.generation ?? '0'),
+    fences: (parsed.fences ?? []).map((f) => ({
+      generation: BigInt(f.generation ?? '0'),
+      txid: BigInt(f.txid ?? '0'),
+    })),
+  };
+}
+
+/** Runs `fn` over `items` with at most `limit` calls in flight, preserving order. */
+async function mapLimit<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]> {
+  const out = new Array<R>(items.length);
+  let next = 0;
+  const worker = async (): Promise<void> => {
+    while (next < items.length) {
+      const i = next++;
+      out[i] = await fn(items[i]);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  return out;
 }
 
+/** Conditional headers of a write. */
+type WriteCondition = { IfNoneMatch: '*' } | { IfMatch: string } | Record<string, never>;
+
 /** Normalized base prefix (no leading/trailing slash, `''` when empty). */
 function normalizePrefix(prefix: string): string {
   return prefix.replace(/^\/+/, '').replace(/\/+$/, '');
@@ -98,6 +286,11 @@ export class S3ReplicaClient implements ReplicaClient {
   readonly prefix: string;
   private client: S3ClientLike;
   private logger: Logger = noopLogger;
+  private generation: bigint | null;
+  /** The higher generation that superseded this client, once observed. */
+  private supersededBy: bigint | null = null;
+  /** Generation of listed objects, keyed by `key` + ETag (objects are immutable). */
+  private readonly objectGenerations = new Map<string, bigint>();
 
   constructor(opts: S3ReplicaClientOptions) {
     if (!opts.bucket) {
@@ -105,19 +298,8 @@ export class S3ReplicaClient implements ReplicaClient {
     }
     this.bucket = opts.bucket;
     this.prefix = normalizePrefix(opts.prefix ?? '');
-    this.client =
-      opts.client ??
-      new S3Client({
-        region: opts.region,
-        endpoint: opts.endpoint,
-        credentials: opts.accessKeyId
-          ? {
-              accessKeyId: opts.accessKeyId,
-              secretAccessKey: opts.secretAccessKey ?? '',
-              sessionToken: opts.sessionToken,
-            }
-          : undefined,
-      });
+    this.generation = opts.generation ?? null;
+    this.client = createClient(opts);
   }
 
   type(): string {
@@ -152,8 +334,166 @@ export class S3ReplicaClient implements ReplicaClient {
     return `${this.levelPrefix(level)}${formatFilename(minTXID, maxTXID)}`;
   }
 
+  /** Returns the key of the generation object. */
+  private generationKey(): string {
+    return `${this.prefix ? `${this.prefix}/` : ''}${GenerationObjectName}`;
+  }
+
+  /** Current fenced-write generation, or `null` when fencing is disabled. */
+  getGeneration(): bigint | null {
+    return this.generation;
+  }
+
+  /** Reads the generation object, or returns `null` if fencing was never used. */
+  async getGenerationState(): Promise<GenerationState | null> {
+    return (await this.readGenerationObject())?.state ?? null;
+  }
+
+  private async readGenerationObject(): Promise<{ state: GenerationState; etag: string } | null> {
+    let res;
+    try {
+      res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: this.generationKey() }));
+    } catch (err) {
+      if (isNotFound(err)) return null;
+      throw err;
+    }
+    const state = decodeGenerationState(await readBody(res.Body));
+    return { state, etag: String(res.ETag ?? '') };
+  }
+
+  /**
+   * Writes the generation object if it was not changed since it was read with
+   * `etag` (`null`: if it does not exist). Returns false on a lost race.
+   */
+  private async writeGenerationObject(state: GenerationState, etag: string | null): Promise<boolean> {
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: this.generationKey(),
+          Body: encodeGenerationState(state),
+          ContentType: 'application/json',
+          ...(etag === null ? { IfNoneMatch: '*' } : { IfMatch: etag }),
+        }),
+      );
+      return true;
+    } catch (err) {
+      if (isPreconditionFailed(err) || isConditionalConflict(err)) return false;
+      throw err;
+    }
+  }
+
+  /** Records that this client was superseded by `generation` and returns the error. */
+  private superseded(generation: bigint): ErrGenerationSuperseded {
+    this.supersededBy = generation;
+    return new ErrGenerationSuperseded(generation);
+  }
+
+  /**
+   * Fails with {@link ErrGenerationSuperseded} if a generation higher than this
+   * client's is registered. A no-op when fencing is disabled.
+   */
+  private async assertGeneration(): Promise<void> {
+    if (this.generation === null) return;
+    if (this.supersededBy !== null) throw new ErrGenerationSuperseded(this.supersededBy);
+    const cur = await this.readGenerationObject();
+    if (cur !== null && cur.state.generation > this.generation) {
+      throw this.superseded(cur.state.generation);
+    }
+  }
+
+  /**
+   * Raises the replica's generation to `generation`, failing with
+   * {@link ErrGenerationSuperseded} if a higher one is already registered, and
+   * records a fence at the replica's latest TXID as read after the
+   * registration. From then on, files of lower generations above that TXID
+   * are ignored by every listing, and uploads and deletions of lower
+   * generations fail once they observe the new generation.
+   *
+   * Call it before restoring: the restore then sees every file that a former
+   * writer can still make part of the replica. Registering the current
+   * generation again is a no-op that keeps its fence.
+   */
+  async registerGeneration(generation: bigint): Promise<void> {
+    // 1. Raise the generation (compare-and-swap on the generation object).
+    for (let attempt = 0; ; attempt++) {
+      if (attempt >= MAX_CONDITIONAL_ATTEMPTS) {
+        throw new Error('S3 replica client: too many conflicts registering the generation');
+      }
+      const cur = await this.readGenerationObject();
+      if (cur !== null && cur.state.generation > generation) {
+        throw new ErrGenerationSuperseded(cur.state.generation);
+      }
+      if (cur !== null && cur.state.generation === generation) break;
+      const next = { generation, fences: cur?.state.fences ?? [] };
+      if (await this.writeGenerationObject(next, cur?.etag ?? null)) break;
+    }
+    this.generation = generation;
+    this.supersededBy = null;
+
+    // 2. Read the latest TXID of the replica. Files of lower generations that
+    //    land from now on and end above it are not part of the replica.
+    let txid = 0n;
+    for (let level = 0; level <= SnapshotLevel; level++) {
+      for await (const info of this.ltxFiles({ level, seek: 0n, useMetadata: false })) {
+        if (info.maxTXID > txid) txid = info.maxTXID;
+      }
+    }
+
+    // 3. Record the fence, unless a higher generation registered meanwhile.
+    for (let attempt = 0; ; attempt++) {
+      if (attempt >= MAX_CONDITIONAL_ATTEMPTS) {
+        throw new Error('S3 replica client: too many conflicts recording the generation fence');
+      }
+      const cur = await this.readGenerationObject();
+      const stored = cur?.state.generation ?? generation;
+      if (stored > generation) throw this.superseded(stored);
+      const fences = cur?.state.fences ?? [];
+      if (fences.some((f) => f.generation === generation)) break;
+      const next = { generation, fences: [...fences, { generation, txid }] };
+      if (await this.writeGenerationObject(next, cur?.etag ?? null)) break;
+    }
+    this.logger.debug('registered generation', { generation, txid });
+  }
+
+  /** Returns the generation of an object (from `ltx-generation`), or null if it is gone. */
+  private async objectGeneration(key: string, etag: string | undefined): Promise<bigint | null> {
+    const cacheKey = etag ? `${key}\n${etag}` : null;
+    if (cacheKey !== null) {
+      const cached = this.objectGenerations.get(cacheKey);
+      if (cached !== undefined) return cached;
+    }
+    let head;
+    try {
+      head = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }));
+    } catch (err) {
+      if (isNotFound(err)) return null;
+      throw err;
+    }
+    const generation = metadataGeneration(head.Metadata);
+    this.rememberGeneration(key, head.ETag ?? etag, generation);
+    return generation;
+  }
+
+  private rememberGeneration(key: string, etag: string | undefined, generation: bigint): void {
+    if (!etag) return;
+    if (this.objectGenerations.size >= GENERATION_CACHE_SIZE) this.objectGenerations.clear();
+    this.objectGenerations.set(`${key}\n${etag}`, generation);
+  }
+
+  /**
+   * Lists the LTX files of a level. When the replica has fences, files ending
+   * above the lowest fence need their generation (a HeadObject request, cached
+   * per key and ETag), and those written by a superseded writer are skipped.
+   */
   async *ltxFiles(opts: LTXFilesOptions): AsyncIterable<FileInfo> {
     const prefix = this.levelPrefix(opts.level);
+    const fences = (await this.readGenerationObject())?.state.fences ?? [];
+    let lowestFence: bigint | null = null;
+    for (const f of fences) {
+      if (lowestFence === null || f.txid < lowestFence) lowestFence = f.txid;
+    }
+
     let token: string | undefined;
     do {
       const res = await this.client.send(
@@ -163,6 +503,7 @@ export class S3ReplicaClient implements ReplicaClient {
           ContinuationToken: token,
         }),
       );
+      const page: Array<{ info: FileInfo; key: string; etag: string | undefined }> = [];
       for (const obj of res.Contents ?? []) {
         if (!obj.Key) continue;
         const name = obj.Key.slice(prefix.length);
@@ -174,13 +515,30 @@ export class S3ReplicaClient implements ReplicaClient {
           continue; // skip non-conforming keys
         }
         if (minTXID < opts.seek) continue;
-        yield {
-          level: opts.level,
-          minTXID,
-          maxTXID,
-          size: obj.Size ?? 0,
-          createdAt: (obj.LastModified ?? new Date(0)).getTime(),
-        };
+        page.push({
+          key: obj.Key,
+          etag: obj.ETag,
+          info: {
+            level: opts.level,
+            minTXID,
+            maxTXID,
+            size: obj.Size ?? 0,
+            createdAt: (obj.LastModified ?? new Date(0)).getTime(),
+          },
+        });
+      }
+
+      let visible: boolean[] = page.map(() => true);
+      if (lowestFence !== null) {
+        const fence = lowestFence;
+        visible = await mapLimit(page, HEAD_CONCURRENCY, async (p) => {
+          if (p.info.maxTXID <= fence) return true;
+          const generation = await this.objectGeneration(p.key, p.etag);
+          return generation !== null && isFenced(fences, generation, p.info.maxTXID);
+        });
+      }
+      for (let i = 0; i < page.length; i++) {
+        if (visible[i]) yield page[i].info;
       }
       token = res.IsTruncated ? (res.NextContinuationToken ?? undefined) : undefined;
     } while (token);
@@ -228,6 +586,10 @@ export class S3ReplicaClient implements ReplicaClient {
 
   async writeLtxFile(opts: WriteLTXFileOptions, body: AsyncIterable<Uint8Array | string>): Promise<FileInfo> {
     const key = this.fileKey(opts.level, opts.minTXID, opts.maxTXID);
+    const generation = this.generation;
+    if (generation !== null && this.supersededBy !== null) {
+      throw new ErrGenerationSuperseded(this.supersededBy);
+    }
 
     // Capture the LTX header (timestamp) from the leading bytes, and stream the
     // rest. Small objects use a single PutObject; large objects (> 5 MiB) are
@@ -241,6 +603,7 @@ export class S3ReplicaClient implements ReplicaClient {
     let size = 0;
 
     let uploadId: string | null = null;
+    let completed = false;
     let partNumber = 0;
     const uploadedParts: Array<{ PartNumber: number; ETag: string }> = [];
 
@@ -258,6 +621,12 @@ export class S3ReplicaClient implements ReplicaClient {
       uploadedParts.push({ PartNumber: partNumber, ETag: res.ETag ?? `etag-${partNumber}` });
     };
 
+    const metadataFor = (timestamp: number): Record<string, string> => {
+      const metadata: Record<string, string> = { 'ltx-timestamp': String(timestamp) };
+      if (generation !== null) metadata[GenerationMetadataKey] = generation.toString();
+      return metadata;
+    };
+
     try {
       for await (const chunk of body) {
         const bytes =
@@ -283,7 +652,7 @@ export class S3ReplicaClient implements ReplicaClient {
             new CreateMultipartUploadCommand({
               Bucket: this.bucket,
               Key: key,
-              Metadata: { 'ltx-timestamp': String(Number(unmarshalHeader(headerBytes!).timestamp)) },
+              Metadata: metadataFor(Number(unmarshalHeader(headerBytes!).timestamp)),
             }),
           );
           uploadId = res.UploadId;
@@ -301,42 +670,69 @@ export class S3ReplicaClient implements ReplicaClient {
         throw new Error('write ltx file: input too short to contain a header');
       }
       const timestamp = Number(unmarshalHeader(headerBytes).timestamp);
-      const metadata = { 'ltx-timestamp': String(timestamp) };
 
-      if (uploadId === null) {
-        // Single PutObject for small objects.
-        const buf = concatBytes(acc);
-        await this.client.send(
-          new PutObjectCommand({
-            Bucket: this.bucket,
-            Key: key,
-            Body: buf,
-            ContentLength: size,
-            Metadata: metadata,
-          }),
-        );
-      } else {
-        // Multipart: upload any remaining bytes as the final part, then complete.
-        if (accLen > 0) await uploadPart(concatBytes(acc));
-        await this.client.send(
+      // Multipart: upload any remaining bytes as the final part.
+      if (uploadId !== null && accLen > 0) {
+        await uploadPart(concatBytes(acc));
+        acc = [];
+        accLen = 0;
+      }
+      const buf = uploadId === null ? concatBytes(acc) : null;
+
+      // Creates the object: a single PutObject for small objects, otherwise
+      // completes the multipart upload.
+      const commit = (condition: WriteCondition): Promise<{ ETag?: string }> => {
+        if (buf !== null) {
+          return this.client.send(
+            new PutObjectCommand({
+              Bucket: this.bucket,
+              Key: key,
+              Body: buf,
+              ContentLength: size,
+              Metadata: metadataFor(timestamp),
+              ...condition,
+            }),
+          );
+        }
+        return this.client.send(
           new CompleteMultipartUploadCommand({
             Bucket: this.bucket,
             Key: key,
-            UploadId: uploadId,
+            UploadId: uploadId!,
             MultipartUpload: { Parts: uploadedParts },
+            ...condition,
           }),
         );
-      }
+      };
 
-      return {
+      const info: FileInfo = {
         level: opts.level,
         minTXID: opts.minTXID,
         maxTXID: opts.maxTXID,
         size,
         createdAt: timestamp,
       };
+
+      if (generation === null) {
+        await commit({});
+        completed = true;
+        return info;
+      }
+
+      // A multipart upload first looks for an existing object, so that the
+      // completion is not expected to fail (S3 may not keep the upload open).
+      const existing = await this.commitFenced(key, opts, generation, commit, uploadId !== null);
+      completed = existing === null;
+
+      // The upload is only acknowledged if this generation is still the latest
+      // after it landed: any generation registered later then sees the file.
+      await this.assertGeneration();
+      return existing ?? info;
     } catch (err) {
-      if (uploadId !== null) {
+      if (err instanceof ErrGenerationSuperseded) throw err;
+      throw new LTXError('write', key, opts.level, opts.minTXID, opts.maxTXID, err);
+    } finally {
+      if (uploadId !== null && !completed) {
         await this.client
           .send(
             new AbortMultipartUploadCommand({
@@ -347,15 +743,83 @@ export class S3ReplicaClient implements ReplicaClient {
           )
           .catch(() => {});
       }
-      throw new LTXError('write', key, opts.level, opts.minTXID, opts.maxTXID, err);
     }
   }
 
+  /**
+   * Creates a fenced LTX object with `If-None-Match: *`. If the key already
+   * exists:
+   *  - written by a higher generation: this client is superseded;
+   *  - written by this generation (a retried upload), or by a lower one but
+   *    still part of the replica: it holds the same transactions, so it is
+   *    kept and its metadata returned;
+   *  - otherwise it is a superseded writer's file, replaced with `If-Match`.
+   * With `probe`, the key is checked with HeadObject before the first write.
+   * Returns null when this call created the object.
+   */
+  private async commitFenced(
+    key: string,
+    opts: WriteLTXFileOptions,
+    generation: bigint,
+    commit: (condition: WriteCondition) => Promise<{ ETag?: string }>,
+    probe: boolean,
+  ): Promise<FileInfo | null> {
+    let condition: WriteCondition = { IfNoneMatch: '*' };
+    for (let attempt = 0; attempt < MAX_CONDITIONAL_ATTEMPTS; attempt++) {
+      if (!probe) {
+        try {
+          const res = await commit(condition);
+          this.rememberGeneration(key, res.ETag, generation);
+          return null;
+        } catch (err) {
+          if (isConditionalConflict(err)) continue;
+          if (!isPreconditionFailed(err)) throw err;
+        }
+      }
+      probe = false;
+
+      let head;
+      try {
+        head = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }));
+      } catch (err) {
+        if (isNotFound(err)) {
+          condition = { IfNoneMatch: '*' }; // deleted meanwhile
+          continue;
+        }
+        throw err;
+      }
+      const existing = metadataGeneration(head.Metadata);
+      if (existing > generation) throw this.superseded(existing);
+      const fences = (await this.readGenerationObject())?.state.fences ?? [];
+      if (existing === generation || isFenced(fences, existing, opts.maxTXID)) {
+        const timestamp = Number(head.Metadata?.['ltx-timestamp']);
+        return {
+          level: opts.level,
+          minTXID: opts.minTXID,
+          maxTXID: opts.maxTXID,
+          size: Number(head.ContentLength ?? 0),
+          createdAt: Number.isFinite(timestamp)
+            ? timestamp
+            : (head.LastModified ?? new Date(0)).getTime(),
+        };
+      }
+      this.logger.debug('replacing ltx file of a superseded writer', { key, generation: existing });
+      condition = { IfMatch: String(head.ETag ?? '') };
+    }
+    throw new Error(`too many conflicting writes to ${key}`);
+  }
+
+  /**
+   * Deletes LTX files. A fenced client first checks that its generation is
+   * still the latest and fails with {@link ErrGenerationSuperseded} otherwise
+   * (a deletion sent just before a new generation registers still applies).
+   */
   async deleteLtxFiles(infos: FileInfo[]): Promise<void> {
     const objects = infos.map((info) => ({
       Key: this.fileKey(info.level, info.minTXID, info.maxTXID),
     }));
     for (let i = 0; i < objects.length; i += 1000) {
+      await this.assertGeneration();
       await this.client.send(
         new DeleteObjectsCommand({
           Bucket: this.bucket,
@@ -365,16 +829,137 @@ export class S3ReplicaClient implements ReplicaClient {
     }
   }
 
+  /**
+   * Deletes every replica under a prefix: every object whose key starts with
+   * `prefix/`, which covers the replica at `prefix` and every replica under
+   * it, generation objects (`generation.json`) and lease objects included
+   * (unlike {@link deleteAll}, which keeps the generation). Resolves with the
+   * sorted prefixes of the deleted replicas: those with an LTX file, a
+   * generation object or a lease object.
+   *
+   * The generation objects are deleted last, so former writers stay fenced
+   * while the files are deleted. Once they are gone, nothing fences those
+   * writers anymore: a writer that is still running can upload again and
+   * create the replica anew. Stop the writers first.
+   *
+   * With {@link S3DeletePrefixOptions.generation}, the deletion is fenced as
+   * the deletions of a fenced client are, on a best-effort basis: the
+   * generation object of each replica is read before deleting its first
+   * object, and again before deleting the generation objects, and the
+   * deletion fails with {@link ErrGenerationSuperseded} if it holds a higher
+   * generation. S3 cannot check one object and delete another atomically, so
+   * a generation registered between a check and a `DeleteObjects` request
+   * does not stop it, and the replicas already processed stay deleted.
+   */
+  static async deletePrefix(opts: S3DeletePrefixOptions): Promise<string[]> {
+    if (!opts.bucket) {
+      throw new Error('S3 replica client: bucket is required');
+    }
+    const prefix = normalizePrefix(opts.prefix ?? '');
+    if (prefix === '') {
+      throw new Error('S3 replica client: deletePrefix requires a non-empty prefix');
+    }
+    const bucket = opts.bucket;
+    const generation = opts.generation ?? null;
+    const ownClient = opts.client === undefined;
+    const client = createClient(opts);
+
+    const replicas = new Set<string>();
+    const checked = new Set<string>();
+    const generationKeys: string[] = [];
+    let pending: Array<{ Key: string }> = [];
+
+    const assertReplica = async (replica: string): Promise<void> => {
+      if (generation === null) return;
+      let res;
+      try {
+        res = await client.send(
+          new GetObjectCommand({ Bucket: bucket, Key: `${replica}/${GenerationObjectName}` }),
+        );
+      } catch (err) {
+        if (isNotFound(err)) return;
+        throw err;
+      }
+      const stored = decodeGenerationState(await readBody(res.Body)).generation;
+      if (stored > generation) throw new ErrGenerationSuperseded(stored);
+    };
+
+    const flush = async (objects: Array<{ Key: string }>): Promise<void> => {
+      for (let i = 0; i < objects.length; i += 1000) {
+        const res = await client.send(
+          new DeleteObjectsCommand({
+            Bucket: bucket,
+            Delete: { Objects: objects.slice(i, i + 1000), Quiet: true },
+          }),
+        );
+        const failed = (res?.Errors ?? []) as Array<{ Key?: string; Code?: string; Message?: string }>;
+        if (failed.length > 0) {
+          const e = failed[0];
+          throw new Error(`S3 replica client: failed to delete ${e.Key}: ${e.Code ?? ''} ${e.Message ?? ''}`.trim());
+        }
+      }
+    };
+
+    try {
+      let token: string | undefined;
+      do {
+        const res = await client.send(
+          new ListObjectsV2Command({ Bucket: bucket, Prefix: `${prefix}/`, ContinuationToken: token }),
+        );
+        for (const obj of (res.Contents ?? []) as Array<{ Key?: string }>) {
+          const key = obj.Key;
+          if (!key) continue;
+          const replica = replicaOfKey(key);
+          if (replica !== null) {
+            replicas.add(replica);
+            if (!checked.has(replica)) {
+              await assertReplica(replica);
+              checked.add(replica);
+            }
+          }
+          if (key.endsWith(`/${GenerationObjectName}`)) {
+            generationKeys.push(key);
+            continue;
+          }
+          pending.push({ Key: key });
+          if (pending.length >= 1000) {
+            await flush(pending);
+            pending = [];
+          }
+        }
+        token = res.IsTruncated ? (res.NextContinuationToken ?? undefined) : undefined;
+      } while (token);
+      await flush(pending);
+
+      for (const key of generationKeys) {
+        await assertReplica(key.slice(0, -(GenerationObjectName.length + 1)));
+      }
+      await flush(generationKeys.map((Key) => ({ Key })));
+    } finally {
+      if (ownClient) (client as S3Client).destroy();
+    }
+    return [...replicas].sort();
+  }
+
+  /**
+   * Deletes every object under the prefix except the generation object, so a
+   * superseded writer stays fenced. A fenced client checks its generation
+   * first, as {@link deleteLtxFiles} does.
+   */
   async deleteAll(): Promise<void> {
     const prefix = this.prefix ? `${this.prefix}/` : '';
+    const generationKey = this.generationKey();
     let token: string | undefined;
     do {
       const res = await this.client.send(
         new ListObjectsV2Command({ Bucket: this.bucket, Prefix: prefix, ContinuationToken: token }),
       );
       const contents = (res.Contents ?? []) as Array<{ Key?: string }>;
-      const objects = contents.filter((o) => o.Key).map((o) => ({ Key: o.Key! }));
+      const objects = contents
+        .filter((o) => o.Key && o.Key !== generationKey)
+        .map((o) => ({ Key: o.Key! }));
       for (let i = 0; i < objects.length; i += 1000) {
+        await this.assertGeneration();
         await this.client.send(
           new DeleteObjectsCommand({
             Bucket: this.bucket,