@platformatic/sqlite-js-stream-s3 0.2.1 → 0.4.0

package/dist/replica-client.js

@@ -10,12 +10,117 @@
  *
  * Creation timestamps are preserved on upload as the `ltx-timestamp` object
  * metadata; listing falls back to the object's `LastModified`.
+ *
+ * The client also supports *fenced writes* (see {@link S3ReplicaClient.registerGeneration}).
+ * S3 cannot check one object and write another in a single request, so fencing
+ * is a protocol built from conditional writes:
+ *
+ * - `{prefix}/generation.json` holds the current generation and a list of
+ *   *fences*. A fence `(generation, txid)` records the latest TXID of the
+ *   replica right after `generation` was registered. Registering is a
+ *   conditional write (`If-Match` / `If-None-Match: *`) that only raises the
+ *   generation; the fence is then added with another conditional write.
+ * - LTX files of a fenced client are written with `If-None-Match: *`, so a TXID
+ *   range is written once per level, and carry their generation in the
+ *   `ltx-generation` metadata. A file with no `ltx-generation` is generation 0.
+ * - Listings ignore a file whose generation is lower than a fence's generation
+ *   and whose `maxTXID` is above that fence's TXID: it was written by a
+ *   superseded writer after the new one read the replica.
+ * - After each upload, a fenced client re-reads the generation object and fails
+ *   with {@link ErrGenerationSuperseded} if a higher generation is registered.
+ *   An upload that resolves was therefore written while its generation was the
+ *   latest, and every later generation sees it.
  */
 import { Readable } from 'node:stream';
 import { S3Client, AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectsCommand, GetObjectCommand, HeadBucketCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, UploadPartCommand, } from '@aws-sdk/client-s3';
-import { HeaderSize, LTXError, formatFilename, noopLogger, parseFilename, unmarshalHeader, } from '@platformatic/sqlite-js-stream';
+import { DefaultLeasePath, ErrGenerationSuperseded, HeaderSize, LTXError, SnapshotLevel, formatFilename, noopLogger, parseFilename, unmarshalHeader, } from '@platformatic/sqlite-js-stream';
+import { isConditionalConflict, isNotFound, isPreconditionFailed, readBody } from "./util.js";
 /** The strategy identifier reported by {@link S3ReplicaClient.type}. */
 export const ReplicaClientType = 's3';
+/** Name of the generation object, relative to the replica prefix. */
+export const GenerationObjectName = 'generation.json';
+/** Object metadata key carrying the generation of a fenced LTX upload. */
+export const GenerationMetadataKey = 'ltx-generation';
+/**
+ * Returns true if a file of `generation` ending at `maxTXID` is part of the
+ * replica: no fence of a higher generation was recorded below `maxTXID`.
+ */
+export function isFenced(fences, generation, maxTXID) {
+    for (const f of fences) {
+        if (f.generation > generation && maxTXID > f.txid)
+            return false;
+    }
+    return true;
+}
+/** Matches the key of an LTX file, capturing the prefix of its replica. */
+const LTX_KEY_RE = /^(.+)\/ltx\/\d+\/[^/]+$/;
+/** Returns the prefix of the replica a key belongs to, or null if none. */
+function replicaOfKey(key) {
+    for (const name of [GenerationObjectName, DefaultLeasePath]) {
+        if (key.endsWith(`/${name}`))
+            return key.slice(0, -(name.length + 1));
+    }
+    return LTX_KEY_RE.exec(key)?.[1] ?? null;
+}
+/** Builds an S3 client from connection options, unless one is injected. */
+function createClient(opts) {
+    return (opts.client ??
+        new S3Client({
+            region: opts.region,
+            endpoint: opts.endpoint,
+            credentials: opts.accessKeyId
+                ? {
+                    accessKeyId: opts.accessKeyId,
+                    secretAccessKey: opts.secretAccessKey ?? '',
+                    sessionToken: opts.sessionToken,
+                }
+                : undefined,
+        }));
+}
+/** Retries of a conditional write that keeps conflicting with concurrent ones. */
+const MAX_CONDITIONAL_ATTEMPTS = 10;
+/** HeadObject requests in flight while resolving the generation of listed files. */
+const HEAD_CONCURRENCY = 16;
+/** Cached generations of listed objects (by key and ETag). */
+const GENERATION_CACHE_SIZE = 10_000;
+/** Parses the `ltx-generation` metadata (0 when absent or invalid). */
+function metadataGeneration(metadata) {
+    const v = metadata?.[GenerationMetadataKey];
+    if (v === undefined || !/^\d+$/.test(v))
+        return 0n;
+    return BigInt(v);
+}
+/** Serializes a generation state (bigints as decimal strings). */
+function encodeGenerationState(state) {
+    return new TextEncoder().encode(JSON.stringify({
+        generation: state.generation.toString(),
+        fences: state.fences.map((f) => ({ generation: f.generation.toString(), txid: f.txid.toString() })),
+    }));
+}
+/** Parses a generation object body. */
+function decodeGenerationState(data) {
+    const parsed = JSON.parse(new TextDecoder().decode(data));
+    return {
+        generation: BigInt(parsed.generation ?? '0'),
+        fences: (parsed.fences ?? []).map((f) => ({
+            generation: BigInt(f.generation ?? '0'),
+            txid: BigInt(f.txid ?? '0'),
+        })),
+    };
+}
+/** Runs `fn` over `items` with at most `limit` calls in flight, preserving order. */
+async function mapLimit(items, limit, fn) {
+    const out = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < items.length) {
+            const i = next++;
+            out[i] = await fn(items[i]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return out;
+}
 /** Normalized base prefix (no leading/trailing slash, `''` when empty). */
 function normalizePrefix(prefix) {
     return prefix.replace(/^\/+/, '').replace(/\/+$/, '');
@@ -41,25 +146,19 @@ export class S3ReplicaClient {
     prefix;
     client;
     logger = noopLogger;
+    generation;
+    /** The higher generation that superseded this client, once observed. */
+    supersededBy = null;
+    /** Generation of listed objects, keyed by `key` + ETag (objects are immutable). */
+    objectGenerations = new Map();
     constructor(opts) {
         if (!opts.bucket) {
             throw new Error('S3 replica client: bucket is required');
         }
         this.bucket = opts.bucket;
         this.prefix = normalizePrefix(opts.prefix ?? '');
-        this.client =
-            opts.client ??
-                new S3Client({
-                    region: opts.region,
-                    endpoint: opts.endpoint,
-                    credentials: opts.accessKeyId
-                        ? {
-                            accessKeyId: opts.accessKeyId,
-                            secretAccessKey: opts.secretAccessKey ?? '',
-                            sessionToken: opts.sessionToken,
-                        }
-                        : undefined,
-                });
+        this.generation = opts.generation ?? null;
+        this.client = createClient(opts);
     }
     type() {
         return ReplicaClientType;
@@ -88,8 +187,169 @@ export class S3ReplicaClient {
     fileKey(level, minTXID, maxTXID) {
         return `${this.levelPrefix(level)}${formatFilename(minTXID, maxTXID)}`;
     }
+    /** Returns the key of the generation object. */
+    generationKey() {
+        return `${this.prefix ? `${this.prefix}/` : ''}${GenerationObjectName}`;
+    }
+    /** Current fenced-write generation, or `null` when fencing is disabled. */
+    getGeneration() {
+        return this.generation;
+    }
+    /** Reads the generation object, or returns `null` if fencing was never used. */
+    async getGenerationState() {
+        return (await this.readGenerationObject())?.state ?? null;
+    }
+    async readGenerationObject() {
+        let res;
+        try {
+            res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: this.generationKey() }));
+        }
+        catch (err) {
+            if (isNotFound(err))
+                return null;
+            throw err;
+        }
+        const state = decodeGenerationState(await readBody(res.Body));
+        return { state, etag: String(res.ETag ?? '') };
+    }
+    /**
+     * Writes the generation object if it was not changed since it was read with
+     * `etag` (`null`: if it does not exist). Returns false on a lost race.
+     */
+    async writeGenerationObject(state, etag) {
+        try {
+            await this.client.send(new PutObjectCommand({
+                Bucket: this.bucket,
+                Key: this.generationKey(),
+                Body: encodeGenerationState(state),
+                ContentType: 'application/json',
+                ...(etag === null ? { IfNoneMatch: '*' } : { IfMatch: etag }),
+            }));
+            return true;
+        }
+        catch (err) {
+            if (isPreconditionFailed(err) || isConditionalConflict(err))
+                return false;
+            throw err;
+        }
+    }
+    /** Records that this client was superseded by `generation` and returns the error. */
+    superseded(generation) {
+        this.supersededBy = generation;
+        return new ErrGenerationSuperseded(generation);
+    }
+    /**
+     * Fails with {@link ErrGenerationSuperseded} if a generation higher than this
+     * client's is registered. A no-op when fencing is disabled.
+     */
+    async assertGeneration() {
+        if (this.generation === null)
+            return;
+        if (this.supersededBy !== null)
+            throw new ErrGenerationSuperseded(this.supersededBy);
+        const cur = await this.readGenerationObject();
+        if (cur !== null && cur.state.generation > this.generation) {
+            throw this.superseded(cur.state.generation);
+        }
+    }
+    /**
+     * Raises the replica's generation to `generation`, failing with
+     * {@link ErrGenerationSuperseded} if a higher one is already registered, and
+     * records a fence at the replica's latest TXID as read after the
+     * registration. From then on, files of lower generations above that TXID
+     * are ignored by every listing, and uploads and deletions of lower
+     * generations fail once they observe the new generation.
+     *
+     * Call it before restoring: the restore then sees every file that a former
+     * writer can still make part of the replica. Registering the current
+     * generation again is a no-op that keeps its fence.
+     */
+    async registerGeneration(generation) {
+        // 1. Raise the generation (compare-and-swap on the generation object).
+        for (let attempt = 0;; attempt++) {
+            if (attempt >= MAX_CONDITIONAL_ATTEMPTS) {
+                throw new Error('S3 replica client: too many conflicts registering the generation');
+            }
+            const cur = await this.readGenerationObject();
+            if (cur !== null && cur.state.generation > generation) {
+                throw new ErrGenerationSuperseded(cur.state.generation);
+            }
+            if (cur !== null && cur.state.generation === generation)
+                break;
+            const next = { generation, fences: cur?.state.fences ?? [] };
+            if (await this.writeGenerationObject(next, cur?.etag ?? null))
+                break;
+        }
+        this.generation = generation;
+        this.supersededBy = null;
+        // 2. Read the latest TXID of the replica. Files of lower generations that
+        //    land from now on and end above it are not part of the replica.
+        let txid = 0n;
+        for (let level = 0; level <= SnapshotLevel; level++) {
+            for await (const info of this.ltxFiles({ level, seek: 0n, useMetadata: false })) {
+                if (info.maxTXID > txid)
+                    txid = info.maxTXID;
+            }
+        }
+        // 3. Record the fence, unless a higher generation registered meanwhile.
+        for (let attempt = 0;; attempt++) {
+            if (attempt >= MAX_CONDITIONAL_ATTEMPTS) {
+                throw new Error('S3 replica client: too many conflicts recording the generation fence');
+            }
+            const cur = await this.readGenerationObject();
+            const stored = cur?.state.generation ?? generation;
+            if (stored > generation)
+                throw this.superseded(stored);
+            const fences = cur?.state.fences ?? [];
+            if (fences.some((f) => f.generation === generation))
+                break;
+            const next = { generation, fences: [...fences, { generation, txid }] };
+            if (await this.writeGenerationObject(next, cur?.etag ?? null))
+                break;
+        }
+        this.logger.debug('registered generation', { generation, txid });
+    }
+    /** Returns the generation of an object (from `ltx-generation`), or null if it is gone. */
+    async objectGeneration(key, etag) {
+        const cacheKey = etag ? `${key}\n${etag}` : null;
+        if (cacheKey !== null) {
+            const cached = this.objectGenerations.get(cacheKey);
+            if (cached !== undefined)
+                return cached;
+        }
+        let head;
+        try {
+            head = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }));
+        }
+        catch (err) {
+            if (isNotFound(err))
+                return null;
+            throw err;
+        }
+        const generation = metadataGeneration(head.Metadata);
+        this.rememberGeneration(key, head.ETag ?? etag, generation);
+        return generation;
+    }
+    rememberGeneration(key, etag, generation) {
+        if (!etag)
+            return;
+        if (this.objectGenerations.size >= GENERATION_CACHE_SIZE)
+            this.objectGenerations.clear();
+        this.objectGenerations.set(`${key}\n${etag}`, generation);
+    }
+    /**
+     * Lists the LTX files of a level. When the replica has fences, files ending
+     * above the lowest fence need their generation (a HeadObject request, cached
+     * per key and ETag), and those written by a superseded writer are skipped.
+     */
     async *ltxFiles(opts) {
         const prefix = this.levelPrefix(opts.level);
+        const fences = (await this.readGenerationObject())?.state.fences ?? [];
+        let lowestFence = null;
+        for (const f of fences) {
+            if (lowestFence === null || f.txid < lowestFence)
+                lowestFence = f.txid;
+        }
         let token;
         do {
             const res = await this.client.send(new ListObjectsV2Command({
@@ -97,6 +357,7 @@ export class S3ReplicaClient {
                 Prefix: prefix,
                 ContinuationToken: token,
             }));
+            const page = [];
             for (const obj of res.Contents ?? []) {
                 if (!obj.Key)
                     continue;
@@ -111,13 +372,31 @@ export class S3ReplicaClient {
                 }
                 if (minTXID < opts.seek)
                     continue;
-                yield {
-                    level: opts.level,
-                    minTXID,
-                    maxTXID,
-                    size: obj.Size ?? 0,
-                    createdAt: (obj.LastModified ?? new Date(0)).getTime(),
-                };
+                page.push({
+                    key: obj.Key,
+                    etag: obj.ETag,
+                    info: {
+                        level: opts.level,
+                        minTXID,
+                        maxTXID,
+                        size: obj.Size ?? 0,
+                        createdAt: (obj.LastModified ?? new Date(0)).getTime(),
+                    },
+                });
+            }
+            let visible = page.map(() => true);
+            if (lowestFence !== null) {
+                const fence = lowestFence;
+                visible = await mapLimit(page, HEAD_CONCURRENCY, async (p) => {
+                    if (p.info.maxTXID <= fence)
+                        return true;
+                    const generation = await this.objectGeneration(p.key, p.etag);
+                    return generation !== null && isFenced(fences, generation, p.info.maxTXID);
+                });
+            }
+            for (let i = 0; i < page.length; i++) {
+                if (visible[i])
+                    yield page[i].info;
             }
             token = res.IsTruncated ? (res.NextContinuationToken ?? undefined) : undefined;
         } while (token);
@@ -162,6 +441,10 @@ export class S3ReplicaClient {
     }
     async writeLtxFile(opts, body) {
         const key = this.fileKey(opts.level, opts.minTXID, opts.maxTXID);
+        const generation = this.generation;
+        if (generation !== null && this.supersededBy !== null) {
+            throw new ErrGenerationSuperseded(this.supersededBy);
+        }
         // Capture the LTX header (timestamp) from the leading bytes, and stream the
         // rest. Small objects use a single PutObject; large objects (> 5 MiB) are
         // uploaded with the multipart API so the whole file is never buffered.
@@ -172,6 +455,7 @@ export class S3ReplicaClient {
         let accLen = 0;
         let size = 0;
         let uploadId = null;
+        let completed = false;
         let partNumber = 0;
         const uploadedParts = [];
         const uploadPart = async (buf) => {
@@ -185,6 +469,12 @@ export class S3ReplicaClient {
             }));
             uploadedParts.push({ PartNumber: partNumber, ETag: res.ETag ?? `etag-${partNumber}` });
         };
+        const metadataFor = (timestamp) => {
+            const metadata = { 'ltx-timestamp': String(timestamp) };
+            if (generation !== null)
+                metadata[GenerationMetadataKey] = generation.toString();
+            return metadata;
+        };
         try {
             for await (const chunk of body) {
                 const bytes = typeof chunk === 'string' ? new TextEncoder().encode(chunk) : Uint8Array.from(chunk);
@@ -206,7 +496,7 @@ export class S3ReplicaClient {
                     const res = await this.client.send(new CreateMultipartUploadCommand({
                         Bucket: this.bucket,
                         Key: key,
-                        Metadata: { 'ltx-timestamp': String(Number(unmarshalHeader(headerBytes).timestamp)) },
+                        Metadata: metadataFor(Number(unmarshalHeader(headerBytes).timestamp)),
                     }));
                     uploadId = res.UploadId;
                     await uploadPart(concatBytes(acc));
@@ -223,39 +513,62 @@ export class S3ReplicaClient {
                 throw new Error('write ltx file: input too short to contain a header');
             }
             const timestamp = Number(unmarshalHeader(headerBytes).timestamp);
-            const metadata = { 'ltx-timestamp': String(timestamp) };
-            if (uploadId === null) {
-                // Single PutObject for small objects.
-                const buf = concatBytes(acc);
-                await this.client.send(new PutObjectCommand({
-                    Bucket: this.bucket,
-                    Key: key,
-                    Body: buf,
-                    ContentLength: size,
-                    Metadata: metadata,
-                }));
+            // Multipart: upload any remaining bytes as the final part.
+            if (uploadId !== null && accLen > 0) {
+                await uploadPart(concatBytes(acc));
+                acc = [];
+                accLen = 0;
             }
-            else {
-                // Multipart: upload any remaining bytes as the final part, then complete.
-                if (accLen > 0)
-                    await uploadPart(concatBytes(acc));
-                await this.client.send(new CompleteMultipartUploadCommand({
+            const buf = uploadId === null ? concatBytes(acc) : null;
+            // Creates the object: a single PutObject for small objects, otherwise
+            // completes the multipart upload.
+            const commit = (condition) => {
+                if (buf !== null) {
+                    return this.client.send(new PutObjectCommand({
+                        Bucket: this.bucket,
+                        Key: key,
+                        Body: buf,
+                        ContentLength: size,
+                        Metadata: metadataFor(timestamp),
+                        ...condition,
+                    }));
+                }
+                return this.client.send(new CompleteMultipartUploadCommand({
                     Bucket: this.bucket,
                     Key: key,
                     UploadId: uploadId,
                     MultipartUpload: { Parts: uploadedParts },
+                    ...condition,
                 }));
-            }
-            return {
+            };
+            const info = {
                 level: opts.level,
                 minTXID: opts.minTXID,
                 maxTXID: opts.maxTXID,
                 size,
                 createdAt: timestamp,
             };
+            if (generation === null) {
+                await commit({});
+                completed = true;
+                return info;
+            }
+            // A multipart upload first looks for an existing object, so that the
+            // completion is not expected to fail (S3 may not keep the upload open).
+            const existing = await this.commitFenced(key, opts, generation, commit, uploadId !== null);
+            completed = existing === null;
+            // The upload is only acknowledged if this generation is still the latest
+            // after it landed: any generation registered later then sees the file.
+            await this.assertGeneration();
+            return existing ?? info;
         }
         catch (err) {
-            if (uploadId !== null) {
+            if (err instanceof ErrGenerationSuperseded)
+                throw err;
+            throw new LTXError('write', key, opts.level, opts.minTXID, opts.maxTXID, err);
+        }
+        finally {
+            if (uploadId !== null && !completed) {
                 await this.client
                     .send(new AbortMultipartUploadCommand({
                     Bucket: this.bucket,
@@ -264,28 +577,209 @@ export class S3ReplicaClient {
                 }))
                     .catch(() => { });
             }
-            throw new LTXError('write', key, opts.level, opts.minTXID, opts.maxTXID, err);
         }
     }
+    /**
+     * Creates a fenced LTX object with `If-None-Match: *`. If the key already
+     * exists:
+     *  - written by a higher generation: this client is superseded;
+     *  - written by this generation (a retried upload), or by a lower one but
+     *    still part of the replica: it holds the same transactions, so it is
+     *    kept and its metadata returned;
+     *  - otherwise it is a superseded writer's file, replaced with `If-Match`.
+     * With `probe`, the key is checked with HeadObject before the first write.
+     * Returns null when this call created the object.
+     */
+    async commitFenced(key, opts, generation, commit, probe) {
+        let condition = { IfNoneMatch: '*' };
+        for (let attempt = 0; attempt < MAX_CONDITIONAL_ATTEMPTS; attempt++) {
+            if (!probe) {
+                try {
+                    const res = await commit(condition);
+                    this.rememberGeneration(key, res.ETag, generation);
+                    return null;
+                }
+                catch (err) {
+                    if (isConditionalConflict(err))
+                        continue;
+                    if (!isPreconditionFailed(err))
+                        throw err;
+                }
+            }
+            probe = false;
+            let head;
+            try {
+                head = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }));
+            }
+            catch (err) {
+                if (isNotFound(err)) {
+                    condition = { IfNoneMatch: '*' }; // deleted meanwhile
+                    continue;
+                }
+                throw err;
+            }
+            const existing = metadataGeneration(head.Metadata);
+            if (existing > generation)
+                throw this.superseded(existing);
+            const fences = (await this.readGenerationObject())?.state.fences ?? [];
+            if (existing === generation || isFenced(fences, existing, opts.maxTXID)) {
+                const timestamp = Number(head.Metadata?.['ltx-timestamp']);
+                return {
+                    level: opts.level,
+                    minTXID: opts.minTXID,
+                    maxTXID: opts.maxTXID,
+                    size: Number(head.ContentLength ?? 0),
+                    createdAt: Number.isFinite(timestamp)
+                        ? timestamp
+                        : (head.LastModified ?? new Date(0)).getTime(),
+                };
+            }
+            this.logger.debug('replacing ltx file of a superseded writer', { key, generation: existing });
+            condition = { IfMatch: String(head.ETag ?? '') };
+        }
+        throw new Error(`too many conflicting writes to ${key}`);
+    }
+    /**
+     * Deletes LTX files. A fenced client first checks that its generation is
+     * still the latest and fails with {@link ErrGenerationSuperseded} otherwise
+     * (a deletion sent just before a new generation registers still applies).
+     */
     async deleteLtxFiles(infos) {
         const objects = infos.map((info) => ({
             Key: this.fileKey(info.level, info.minTXID, info.maxTXID),
         }));
         for (let i = 0; i < objects.length; i += 1000) {
+            await this.assertGeneration();
             await this.client.send(new DeleteObjectsCommand({
                 Bucket: this.bucket,
                 Delete: { Objects: objects.slice(i, i + 1000), Quiet: true },
             }));
         }
     }
+    /**
+     * Deletes every replica under a prefix: every object whose key starts with
+     * `prefix/`, which covers the replica at `prefix` and every replica under
+     * it, generation objects (`generation.json`) and lease objects included
+     * (unlike {@link deleteAll}, which keeps the generation). Resolves with the
+     * sorted prefixes of the deleted replicas: those with an LTX file, a
+     * generation object or a lease object.
+     *
+     * The generation objects are deleted last, so former writers stay fenced
+     * while the files are deleted. Once they are gone, nothing fences those
+     * writers anymore: a writer that is still running can upload again and
+     * create the replica anew. Stop the writers first.
+     *
+     * With {@link S3DeletePrefixOptions.generation}, the deletion is fenced as
+     * the deletions of a fenced client are, on a best-effort basis: the
+     * generation object of each replica is read before deleting its first
+     * object, and again before deleting the generation objects, and the
+     * deletion fails with {@link ErrGenerationSuperseded} if it holds a higher
+     * generation. S3 cannot check one object and delete another atomically, so
+     * a generation registered between a check and a `DeleteObjects` request
+     * does not stop it, and the replicas already processed stay deleted.
+     */
+    static async deletePrefix(opts) {
+        if (!opts.bucket) {
+            throw new Error('S3 replica client: bucket is required');
+        }
+        const prefix = normalizePrefix(opts.prefix ?? '');
+        if (prefix === '') {
+            throw new Error('S3 replica client: deletePrefix requires a non-empty prefix');
+        }
+        const bucket = opts.bucket;
+        const generation = opts.generation ?? null;
+        const ownClient = opts.client === undefined;
+        const client = createClient(opts);
+        const replicas = new Set();
+        const checked = new Set();
+        const generationKeys = [];
+        let pending = [];
+        const assertReplica = async (replica) => {
+            if (generation === null)
+                return;
+            let res;
+            try {
+                res = await client.send(new GetObjectCommand({ Bucket: bucket, Key: `${replica}/${GenerationObjectName}` }));
+            }
+            catch (err) {
+                if (isNotFound(err))
+                    return;
+                throw err;
+            }
+            const stored = decodeGenerationState(await readBody(res.Body)).generation;
+            if (stored > generation)
+                throw new ErrGenerationSuperseded(stored);
+        };
+        const flush = async (objects) => {
+            for (let i = 0; i < objects.length; i += 1000) {
+                const res = await client.send(new DeleteObjectsCommand({
+                    Bucket: bucket,
+                    Delete: { Objects: objects.slice(i, i + 1000), Quiet: true },
+                }));
+                const failed = (res?.Errors ?? []);
+                if (failed.length > 0) {
+                    const e = failed[0];
+                    throw new Error(`S3 replica client: failed to delete ${e.Key}: ${e.Code ?? ''} ${e.Message ?? ''}`.trim());
+                }
+            }
+        };
+        try {
+            let token;
+            do {
+                const res = await client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: `${prefix}/`, ContinuationToken: token }));
+                for (const obj of (res.Contents ?? [])) {
+                    const key = obj.Key;
+                    if (!key)
+                        continue;
+                    const replica = replicaOfKey(key);
+                    if (replica !== null) {
+                        replicas.add(replica);
+                        if (!checked.has(replica)) {
+                            await assertReplica(replica);
+                            checked.add(replica);
+                        }
+                    }
+                    if (key.endsWith(`/${GenerationObjectName}`)) {
+                        generationKeys.push(key);
+                        continue;
+                    }
+                    pending.push({ Key: key });
+                    if (pending.length >= 1000) {
+                        await flush(pending);
+                        pending = [];
+                    }
+                }
+                token = res.IsTruncated ? (res.NextContinuationToken ?? undefined) : undefined;
+            } while (token);
+            await flush(pending);
+            for (const key of generationKeys) {
+                await assertReplica(key.slice(0, -(GenerationObjectName.length + 1)));
+            }
+            await flush(generationKeys.map((Key) => ({ Key })));
+        }
+        finally {
+            if (ownClient)
+                client.destroy();
+        }
+        return [...replicas].sort();
+    }
+    /**
+     * Deletes every object under the prefix except the generation object, so a
+     * superseded writer stays fenced. A fenced client checks its generation
+     * first, as {@link deleteLtxFiles} does.
+     */
     async deleteAll() {
         const prefix = this.prefix ? `${this.prefix}/` : '';
+        const generationKey = this.generationKey();
         let token;
         do {
             const res = await this.client.send(new ListObjectsV2Command({ Bucket: this.bucket, Prefix: prefix, ContinuationToken: token }));
             const contents = (res.Contents ?? []);
-            const objects = contents.filter((o) => o.Key).map((o) => ({ Key: o.Key }));
+            const objects = contents
+                .filter((o) => o.Key && o.Key !== generationKey)
+                .map((o) => ({ Key: o.Key }));
             for (let i = 0; i < objects.length; i += 1000) {
+                await this.assertGeneration();
                 await this.client.send(new DeleteObjectsCommand({
                     Bucket: this.bucket,
                     Delete: { Objects: objects.slice(i, i + 1000), Quiet: true },