lambder 7.3.1 → 8.1.1

package/dist/stores/LambderDdbCache.js

@@ -1,25 +1,39 @@
 import { createDynamoClientLoader, isConditionalCheckFailure } from "./LambderDdbSdk.js";
 import { getCrypto } from "../shared/util/LambderNodeModules.js";
+import { LambderExpiringMap } from "../shared/util/LambderExpiringMap.js";
 import { assertPositiveInteger } from "../shared/util/LambderOptionChecks.js";
 import { compressText, restoreText, LambderCompressionError } from "../shared/wire/LambderCompressionCodec.js";
 import { resolveCompressionOption, } from "../shared/wire/LambderCompressionOption.js";
 import { LRUCache } from "lru-cache";
-const DEFAULT_TTL_SECONDS = 365 * 24 * 60 * 60;
+import { cacheMemoryKeyOf, decodeCacheSortKey, encodeCacheSortKey, normalizeCacheKey, normalizeCachePartition, } from "./LambderCacheKeys.js";
+import { DEFAULT_MAX_VALUE_BYTES, DEFAULT_TTL_SECONDS, resolveCacheTtlSeconds, resolveGetOrSetOptions, serializeCacheValue, } from "./LambderCacheValues.js";
+import { LambderCacheFiller } from "./LambderCacheFiller.js";
 const DEFAULT_CHUNK_BYTES = 350 * 1024;
 const MAX_SAFE_CHUNK_BYTES = 380 * 1024;
-const DEFAULT_MAX_VALUE_BYTES = 32 * 1024 * 1024;
 const DEFAULT_MEMORY_BYTES = 16 * 1024 * 1024;
+/**
+ * What one memory-layer entry costs beyond its bytes and its key: the entry
+ * object and the LRU's own bookkeeping for it. Counted so that many small
+ * values fill the budget at the rate they actually use memory.
+ */
+const MEMORY_ENTRY_OVERHEAD_BYTES = 256;
 const META_SORT_KEY = "meta";
-const LOCK_SORT_KEY = "lock";
 const CHUNK_SORT_KEY_PREFIX = "chunk#";
 /** Item-key prefix that separates entries addressed with a sort key from plain-key entries sharing the partition. */
 const SORT_KEY_MARKER = "sk#";
-/** Budget for one encoded sort key, leaving room for the marker and the longest item suffix inside DynamoDB's 1024-byte range key limit. */
-const MAX_SORT_KEY_BYTES = 900;
 const BATCH_WRITE_LIMIT = 25;
 const MAX_BATCH_RETRIES = 8;
 /** Every value compressed by default; see the `compression` option. */
 const COMPRESSION_DEFAULTS = { minBytes: 0, quality: 5 };
+/**
+ * How long this instance reads a key it wrote with consistent reads, in
+ * seconds (whole ones, so from four to five). DynamoDB's replicas apply a
+ * write well within that, and the margin costs little: only the keys this
+ * instance wrote are read at a consistent read's price.
+ */
+const RECENT_WRITE_SECONDS = 5;
+/** Keys, and partitions, remembered as recently written at most; past it the ones closest to expiring go first (see LambderExpiringMap). */
+const RECENT_WRITE_MAX_ENTRIES = 1_000;
 /**
  * A stored entry that cannot be trusted: chunks that do not add up to what
  * the manifest describes, bytes that fail its checksum, or a restore the
@@ -29,9 +43,9 @@ const COMPRESSION_DEFAULTS = { minBytes: 0, quality: 5 };
  * A type rather than "anything thrown while reading the entry", because the
  * two are treated in opposite ways: a corrupt entry is dropped so the next
  * reader refills it, while a throttled or failed Query is the table being
- * busy. Deleting a healthy 2MB entry over one throttle orphans its chunks
- * until their TTL and sends every later reader to the origin, which is the
- * load the cache exists to absorb.
+ * busy. Deleting a healthy 2MB entry over one throttle would orphan its
+ * chunks until their TTL and send every later reader to the origin, the load
+ * the cache exists to absorb.
  */
 class LambderCacheIntegrityError extends Error {
     constructor(message) {
@@ -39,22 +53,17 @@ class LambderCacheIntegrityError extends Error {
         this.name = "LambderCacheIntegrityError";
     }
 }
+/** A read that proves the entry wrong, as opposed to the table answering badly: only such an entry is ever dropped. */
+const isEntryFault = (error) => error instanceof LambderCacheIntegrityError || error instanceof LambderCompressionError;
 /**
- * `#` separates the store's own item-key segments, so a caller's `#` is
- * escaped rather than refused: `~` becomes `~0` and `#` becomes `~1`. An
- * encoded sort key therefore never contains a bare `#`, which keeps
- * `<encoded>#` an unambiguous boundary for prefix queries. Escaping is
- * per-character, so a prefix of the raw key stays a prefix of the encoded
- * one; only the sort ORDER of keys that contain `#` or `~` shifts, since
- * both encode into the `~` range.
+ * Whether a manifest item holds an entry: a value whose expiresAt has not
+ * passed. A fill's lease holds no value, and an expired value the table's TTL
+ * has not removed yet is a miss to every read, so neither is one.
  */
-const encodeSortKey = (value) => value.replace(/~/g, "~0").replace(/#/g, "~1");
-const decodeSortKey = (value) => value.replace(/~([01])/g, (_match, code) => code === "0" ? "~" : "#");
+const holdsLiveValue = (item, nowSeconds) => item?.version?.S !== undefined && Number(item.expiresAt?.N) > nowSeconds;
 // Node builtins are loaded lazily through LambderNodeModules so this module can
 // sit in a frontend bundle's import graph (via the package root) without
-// breaking; using the cache at runtime still requires Node. Brotli helpers
-// are shared with LambderDdbIdempotencyStore and LambderDdbSessionStore via
-// ../shared/wire/LambderCompressionCodec.js.
+// breaking; using the cache at runtime still requires Node.
 const requireCrypto = async () => {
     const crypto = await getCrypto();
     if (!crypto)
@@ -70,17 +79,93 @@ const randomUUID = async () => {
     return crypto.randomUUID();
 };
 const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
+/**
+ * What an instance's reads need to know about its own writes (set, delete,
+ * deletePartition, a fill's publish), so that its memory layer never keeps a
+ * value one of them replaced.
+ *
+ * The table does not answer in the order it applied writes, and an
+ * eventually consistent read answers from a replica that may not have
+ * applied the latest ones. So a value a read fetched can be one a write of
+ * this instance had already replaced, in two ways:
+ *
+ * - The read overlapped the write. `watch` counts, per key, the writes that
+ *   finish while an operation on the key is in flight, and the operation
+ *   keeps what it got only when none did. Only keys with an operation in
+ *   flight are held, so a write of one key never costs another key its copy.
+ * - The read began after the write finished and reached a replica that had
+ *   not applied it. For a few seconds after each write, `wroteRecently`
+ *   answers true for the key (and for every key of a partition that
+ *   deletePartition dropped), and the read goes to the leader instead.
+ *
+ * A write settles its key's memory copy as it finishes, keeping its own value
+ * or dropping the copy, so whatever an earlier read kept cannot outlive it.
+ */
+class LocalWriteLedger {
+    /** Per memory key with an operation in flight: how many there are, and how many writes of the key have finished since the first began. */
+    keysInFlight = new Map();
+    recentKeys;
+    recentPartitions;
+    now;
+    constructor(now) {
+        this.now = now;
+        this.recentKeys = new LambderExpiringMap({ now, maxEntries: RECENT_WRITE_MAX_ENTRIES });
+        this.recentPartitions = new LambderExpiringMap({ now, maxEntries: RECENT_WRITE_MAX_ENTRIES });
+    }
+    /**
+     * Runs `operation` on one key, handing it `overtaken`, which answers
+     * whether a write of the key (a deletePartition of its partition
+     * included) has finished since the operation began.
+     */
+    async watch(memoryKey, operation) {
+        const activity = this.keysInFlight.get(memoryKey) ?? { operations: 0, finishedWrites: 0 };
+        this.keysInFlight.set(memoryKey, activity);
+        activity.operations += 1;
+        const startedAt = activity.finishedWrites;
+        try {
+            return await operation(() => activity.finishedWrites !== startedAt);
+        }
+        finally {
+            activity.operations -= 1;
+            if (activity.operations === 0)
+                this.keysInFlight.delete(memoryKey);
+        }
+    }
+    /** A write of one key has finished, landed or failed: the table may hold it either way. */
+    recordKeyWrite(memoryKey) {
+        const activity = this.keysInFlight.get(memoryKey);
+        if (activity)
+            activity.finishedWrites += 1;
+        this.recentKeys.set(memoryKey, true, this.recentUntil());
+    }
+    /** A deletePartition has finished, landed or failed: a write of every key in the partition. */
+    recordPartitionWrite(partition) {
+        const prefix = cacheMemoryKeyOf(partition, "");
+        for (const [memoryKey, activity] of this.keysInFlight) {
+            if (memoryKey.startsWith(prefix))
+                activity.finishedWrites += 1;
+        }
+        this.recentPartitions.set(partition, true, this.recentUntil());
+    }
+    /** Whether this instance wrote the entry a moment ago, recently enough that a replica may not have applied the write yet. */
+    wroteRecently(address) {
+        return this.recentKeys.get(address.memoryKey) !== undefined || this.recentPartitions.get(address.partition) !== undefined;
+    }
+    recentUntil() {
+        return Math.floor(this.now() / 1000) + RECENT_WRITE_SECONDS;
+    }
+}
 /**
  * Persistent JSON cache backed by DynamoDB.
  *
  * Values are Brotli-compressed by default (`compression` option; the manifest
  * records each value's encoding, so the option can be switched on a live
  * table). Values within the safe DynamoDB item budget are stored directly in
- * the manifest for a single-request read; larger values are
- * split into versioned binary chunks. A manifest is written only after every
- * chunk succeeds, so readers see either the previous complete version or the
- * new complete version. DynamoDB TTL is cleanup only; every read also checks
- * expiresAt because TTL deletion can lag.
+ * the manifest for a single-request read; larger values are split into
+ * versioned binary chunks. A manifest is written only after every chunk
+ * succeeds, so readers see either the previous complete version or the new
+ * one. DynamoDB TTL is cleanup only; every read also checks expiresAt because
+ * TTL deletion can lag.
  *
  * Table shape: string hash key `pk`, string range key `sk`, TTL on
  * `expiresAt`. Items are prefixed `CACHE#<namespace>#` by default, so the
@@ -89,10 +174,22 @@ const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, mil
  *
  * A key may also be a `{ pk, sk }` pair, which groups entries under one
  * partition so `deletePartition` and `listSortKeys` can work on the group
- * without knowing its members. Plain-string keys keep the exact item layout
- * they have always had (`meta`, `lock`, `chunk#...`), and grouped entries
- * live beside them under `sk#<encoded sort key>#...`, so both forms can
- * share a partition and a live table needs no migration.
+ * without knowing its members. Only the `pk` part is hashed into the
+ * DynamoDB partition key; the sort key is stored readable, which makes
+ * prefix queries possible. Plain-string keys use the bare item keys (`meta`,
+ * `chunk#...`) and grouped entries live beside them under
+ * `sk#<encoded sort key>#...`, so both forms can share a partition.
+ *
+ * getOrSet's fill lease lives on the entry's manifest item (see takeLease),
+ * and a fill publishes only while its lease is still there. `set` replaces
+ * the manifest, and `delete` and `deletePartition` remove it, lease and all,
+ * so a fill that started before any of them never stores over it. Both find
+ * what to remove on the leader (`delete` by the manifest's own key,
+ * `deletePartition` with a consistent Query), since a replica may not have
+ * seen a value or a lease the table accepted a moment before.
+ *
+ * Implements LambderCache, so code typed against the interface runs over
+ * LambderMemoryCache in a test.
  */
 export class LambderDdbCache {
     tableName;
@@ -105,8 +202,11 @@ export class LambderDdbCache {
     compression;
     maxValueBytes;
     memory;
-    inFlight = new Map();
+    /** getOrSet's single-flight and fail-open, shared with LambderMemoryCache (see LambderCacheFiller). */
+    filler;
     now;
+    /** This instance's own writes, as its reads and its memory layer need to know them (see LocalWriteLedger). */
+    localWrites;
     constructor(options) {
         if (!options.tableName.trim())
             throw new Error("tableName is required");
@@ -128,13 +228,16 @@ export class LambderDdbCache {
             ? null
             : new LRUCache({
                 maxSize: assertPositiveInteger(memoryMaxBytes, "memoryMaxBytes"),
-                sizeCalculation: (entry) => entry.stored.length,
+                // The key is a JS string, two bytes a character.
+                sizeCalculation: (entry, key) => entry.stored.byteLength + key.length * 2 + MEMORY_ENTRY_OVERHEAD_BYTES,
             });
         this.now = options.now ?? (() => Date.now());
+        this.localWrites = new LocalWriteLedger(this.now);
         this.ready = createDynamoClientLoader({ user: "LambderDdbCache", region: options.region, client: options.client });
+        this.filler = new LambderCacheFiller(`DynamoDB cache failed open in ${this.namespace}`);
     }
     async get(key) {
-        return await this.getByAddress(this.normalizeKey(key));
+        return await this.getByAddress(normalizeCacheKey(key));
     }
     async getByAddress(address) {
         const cached = this.memory?.get(address.memoryKey);
@@ -149,58 +252,112 @@ export class LambderDdbCache {
         }
         if (cached)
             this.memory?.delete(address.memoryKey);
-        const pk = await this.partitionKey(address.partition);
-        const manifest = await this.readManifest(pk, address);
-        if (!manifest || manifest.expiresAt <= nowSeconds)
-            return undefined;
-        try {
-            const stored = manifest.inlineData ?? await this.readChunks(pk, address, manifest);
-            if (stored.length !== manifest.storedBytes) {
-                throw new LambderCacheIntegrityError("stored byte length does not match manifest");
+        return await this.localWrites.watch(address.memoryKey, async (overtaken) => {
+            const pk = await this.partitionKey(address.partition);
+            // A key this instance wrote a moment ago is read from the leader:
+            // a replica may not have applied that write yet, and the memory
+            // layer would keep what it had instead (see LocalWriteLedger).
+            const consistent = this.localWrites.wroteRecently(address);
+            const manifest = await this.readManifest(pk, address, consistent);
+            if (!manifest || manifest.expiresAt <= nowSeconds)
+                return undefined;
+            try {
+                return await this.readEntry(pk, address, manifest, consistent, overtaken);
             }
-            if (await sha256(stored) !== manifest.checksum) {
-                throw new LambderCacheIntegrityError("stored checksum does not match manifest");
+            catch (error) {
+                // Only an entry this read can prove wrong is dropped. Anything
+                // else is the table answering badly and propagates, like the
+                // manifest read's errors, for getOrSet's fail-open to handle as
+                // the infrastructure failure it is.
+                if (!isEntryFault(error))
+                    throw error;
             }
-            const json = await this.decode(stored, manifest.encoding, manifest.uncompressedBytes);
-            const parsed = JSON.parse(json);
-            this.remember(address.memoryKey, stored, manifest.encoding, manifest.uncompressedBytes, manifest.expiresAt);
-            return parsed;
+            // One read is not proof yet: a replica can hold a manifest before
+            // its chunks, and a newer write can replace the version and delete
+            // its chunks while they are read. One consistent read of both
+            // decides, and only what fails that is corrupt.
+            const current = await this.readManifest(pk, address, true);
+            if (!current || current.expiresAt <= this.nowSeconds())
+                return undefined;
+            return await this.readLeaderEntry(pk, address, current, overtaken);
+        });
+    }
+    /**
+     * The value a manifest the leader answered describes, its chunks read
+     * consistently, or undefined when it fails that read: then it is corrupt,
+     * and its manifest is dropped. `overtaken` comes from the read's watch
+     * (see LocalWriteLedger).
+     */
+    async readLeaderEntry(pk, address, manifest, overtaken) {
+        try {
+            return await this.readEntry(pk, address, manifest, true, overtaken);
         }
         catch (error) {
-            // Only an entry this read can prove wrong is dropped. Everything
-            // else is the table answering badly, which is the manifest read's
-            // own behaviour one line up: it propagates, and getOrSet's
-            // fail-open handles it as the infrastructure failure it is.
-            if (!(error instanceof LambderCacheIntegrityError || error instanceof LambderCompressionError))
+            if (!isEntryFault(error))
                 throw error;
             await this.invalidateManifest(pk, address, manifest.version);
             console.warn(`Ignoring corrupt DynamoDB cache entry in ${this.namespace}`, error);
             return undefined;
         }
     }
+    /**
+     * The value a manifest describes, checked against it and kept in the
+     * memory layer unless a write of the key finished while it was read
+     * (`overtaken`, see LocalWriteLedger).
+     */
+    async readEntry(pk, address, manifest, consistent, overtaken) {
+        const stored = manifest.inlineData ?? await this.readChunks(pk, address, manifest, consistent);
+        if (stored.length !== manifest.storedBytes) {
+            throw new LambderCacheIntegrityError("stored byte length does not match manifest");
+        }
+        if (await sha256(stored) !== manifest.checksum) {
+            throw new LambderCacheIntegrityError("stored checksum does not match manifest");
+        }
+        const json = await this.decode(stored, manifest.encoding, manifest.uncompressedBytes);
+        const parsed = JSON.parse(json);
+        if (!overtaken()) {
+            this.remember(address.memoryKey, stored, manifest.encoding, manifest.uncompressedBytes, manifest.expiresAt);
+        }
+        return parsed;
+    }
     async has(key) {
-        const address = this.normalizeKey(key);
+        const address = normalizeCacheKey(key);
         const cached = this.memory?.get(address.memoryKey);
         const nowSeconds = this.nowSeconds();
         if (cached?.expiresAt && cached.expiresAt > nowSeconds)
             return true;
         if (cached)
             this.memory?.delete(address.memoryKey);
-        const manifest = await this.readManifest(await this.partitionKey(address.partition), address);
+        // From the leader for a key this instance wrote a moment ago, as getByAddress reads it.
+        const manifest = await this.readManifest(await this.partitionKey(address.partition), address, this.localWrites.wroteRecently(address));
         return !!manifest && manifest.expiresAt > nowSeconds;
     }
     async set(key, value, options = {}) {
-        return await this.setByAddress(this.normalizeKey(key), value, options);
+        const address = normalizeCacheKey(key);
+        const ttlSeconds = resolveCacheTtlSeconds(options.ttlSeconds, this.defaultTtlSeconds);
+        this.filler.supersedeFill(address.memoryKey);
+        await this.setByAddress(address, value, ttlSeconds);
     }
-    async setByAddress(address, value, options) {
-        const ttlSeconds = assertPositiveInteger(options.ttlSeconds ?? this.defaultTtlSeconds, "ttlSeconds");
-        const json = JSON.stringify(value);
-        if (json === undefined)
-            throw new Error("Cache value must be JSON-serializable");
-        const input = Buffer.from(json, "utf8");
-        if (input.length > this.maxValueBytes) {
-            throw new Error(`Cache value exceeds maxValueBytes (${input.length} > ${this.maxValueBytes})`);
-        }
+    /**
+     * Stores the value and hands back what was stored, the parse of its
+     * JSON: what every later read answers too.
+     *
+     * With `leaseOwner` it is a fill publishing under its lease, and the
+     * manifest is written only while that lease is still on it. A set,
+     * delete or deletePartition since the lease was taken replaced or
+     * removed it, and so did a waiter that took it over once it lapsed; the
+     * loader may have read its source before any of those, so its value must
+     * not land over them. A refused fill deletes the chunks it wrote, which
+     * belong to no manifest, and hands its value back uncached. A write
+     * winning is the design and is not logged; a takeover is logged, since it
+     * means the loader ran past `leaseSeconds`, and while every fill does,
+     * each is refused by the next takeover and the entry never fills. The
+     * cure is a lease longer than the loader's worst case, which only the
+     * caller can give.
+     */
+    async setByAddress(address, value, ttlSeconds, leaseOwner) {
+        const { json, utf8 } = serializeCacheValue(value, this.maxValueBytes);
+        const input = Buffer.from(utf8.buffer, utf8.byteOffset, utf8.byteLength);
         const brotli = this.compression && input.length >= this.compression.minBytes ? this.compression : null;
         const encoding = brotli ? "br" : "identity";
         const stored = brotli ? await compressText(input, "br", brotli.quality) : input;
@@ -227,188 +384,352 @@ export class LambderDdbCache {
                 },
             },
         }));
-        await this.batchWrite(writes);
-        const { client, sdk } = await this.ready();
-        await client.send(new sdk.PutItemCommand({
-            TableName: this.tableName,
-            Item: {
-                pk: { S: pk },
-                sk: { S: this.itemSortKey(address, META_SORT_KEY) },
-                version: { S: version },
-                chunkCount: { N: String(chunks.length) },
-                storedBytes: { N: String(stored.length) },
-                uncompressedBytes: { N: String(input.length) },
-                checksum: { S: await sha256(stored) },
-                encoding: { S: encoding },
-                createdAt: { N: String(this.nowSeconds()) },
-                expiresAt: { N: String(expiresAt) },
-                ...(inline ? { data: { B: stored } } : {}),
-            },
-        }));
-        this.remember(address.memoryKey, stored, encoding, input.length, expiresAt);
+        const publish = await this.localWrites.watch(address.memoryKey, async (overtaken) => {
+            let kept = false;
+            try {
+                await this.batchWrite(writes);
+                const { client, sdk } = await this.ready();
+                let replaced;
+                let refused = false;
+                try {
+                    const response = await client.send(new sdk.PutItemCommand({
+                        TableName: this.tableName,
+                        ReturnValues: "ALL_OLD",
+                        Item: {
+                            pk: { S: pk },
+                            sk: { S: this.itemSortKey(address, META_SORT_KEY) },
+                            version: { S: version },
+                            chunkCount: { N: String(chunks.length) },
+                            storedBytes: { N: String(stored.length) },
+                            uncompressedBytes: { N: String(input.length) },
+                            checksum: { S: await sha256(stored) },
+                            encoding: { S: encoding },
+                            createdAt: { N: String(this.nowSeconds()) },
+                            expiresAt: { N: String(expiresAt) },
+                            ...(inline ? { data: { B: stored } } : {}),
+                        },
+                        ...(leaseOwner === undefined ? {} : {
+                            ConditionExpression: "#leaseOwner = :leaseOwner",
+                            ExpressionAttributeNames: { "#leaseOwner": "leaseOwner" },
+                            ExpressionAttributeValues: { ":leaseOwner": { S: leaseOwner } },
+                            ReturnValuesOnConditionCheckFailure: "ALL_OLD",
+                        }),
+                    }));
+                    replaced = response.Attributes;
+                }
+                catch (error) {
+                    if (leaseOwner === undefined || !isConditionalCheckFailure(error))
+                        throw error;
+                    const current = error.Item;
+                    // Refused over this very manifest when the SDK retried an
+                    // attempt that had already landed: published, not refused.
+                    refused = current?.version?.S !== version;
+                    if (refused && current?.leaseOwner?.S !== undefined) {
+                        console.warn(`LambderDdbCache: a fill in ${this.namespace} ran past its lease and another container took the lease over, so its value was not stored; give getOrSet a leaseSeconds longer than its loader takes.`);
+                    }
+                }
+                // Kept only when no other write of this key finished while
+                // this one was in flight (see LocalWriteLedger).
+                if (!refused && !overtaken()) {
+                    this.remember(address.memoryKey, stored, encoding, input.length, expiresAt);
+                    kept = true;
+                }
+                return { refused, replaced };
+            }
+            finally {
+                // Settled as it lands, whatever happened: its own value in
+                // memory, or no copy of the key at all.
+                if (!kept)
+                    this.memory?.delete(address.memoryKey);
+                this.localWrites.recordKeyWrite(address.memoryKey);
+            }
+        });
+        if (publish.refused) {
+            await this.deleteChunks(pk, address, version, chunks.length, "a refused fill's");
+        }
+        else {
+            await this.deleteReplacedChunks(pk, address, publish.replaced, version);
+        }
+        return JSON.parse(json);
+    }
+    /**
+     * The chunks of the version a manifest write just replaced, which nothing
+     * reads any more and which would otherwise sit in the table until their
+     * TTL: a large value refreshed hourly leaves a copy an hour. `ownVersion`
+     * is the writer's own, which an SDK retry of a write that had already
+     * landed reports as the replaced one.
+     */
+    async deleteReplacedChunks(pk, address, previous, ownVersion) {
+        const previousVersion = previous?.version?.S;
+        const chunkCount = Number(previous?.chunkCount?.N);
+        if (!previousVersion || previousVersion === ownVersion || !Number.isSafeInteger(chunkCount) || chunkCount <= 0)
+            return;
+        const count = Math.min(chunkCount, Math.ceil(this.maxValueBytes / this.chunkBytes));
+        await this.deleteChunks(pk, address, previousVersion, count, "a replaced value's");
     }
+    /**
+     * One version's chunk items, deleted. The write they belonged to is
+     * settled whether or not this succeeds, and the chunks carry their own
+     * TTL, so a failure here is logged, not the caller's.
+     */
+    async deleteChunks(pk, address, version, count, whose) {
+        try {
+            await this.batchWrite(Array.from({ length: count }, (_, index) => ({
+                DeleteRequest: { Key: { pk: { S: pk }, sk: { S: this.chunkSortKey(address, version, index) } } },
+            })));
+        }
+        catch (error) {
+            console.warn(`Failed to delete ${whose} chunks from the DynamoDB cache in ${this.namespace}`, error);
+        }
+    }
+    /**
+     * Removes the entry's manifest item, whatever it holds (a value, or a
+     * fill's lease, whose publish is then refused), and its chunks. True when
+     * the manifest held a live value: a lease or an expired value the table's
+     * TTL has not removed yet is no entry.
+     */
     async delete(key) {
-        const address = this.normalizeKey(key);
-        const pk = await this.partitionKey(address.partition);
+        const address = normalizeCacheKey(key);
+        this.filler.supersedeFill(address.memoryKey);
         this.memory?.delete(address.memoryKey);
-        // A grouped entry owns one contiguous item range; a plain-string one
-        // owns the bare item keys, so it must leave any grouped entries
-        // sharing its partition alone.
-        const prefix = this.entryItemPrefix(address);
-        const items = await this.queryItems(pk, { prefix, projection: "#pk, #sk" });
-        const owned = prefix ? items : items.filter((item) => !item.sk?.S?.startsWith(SORT_KEY_MARKER));
-        await this.deleteItems(owned);
-        return owned.length > 0;
+        try {
+            const pk = await this.partitionKey(address.partition);
+            const { client, sdk } = await this.ready();
+            // By its key, so the leader removes what it holds now: a value or
+            // a lease another container wrote a moment ago, which a Query
+            // could miss on a replica that has not seen it yet.
+            const { Attributes: removed } = await client.send(new sdk.DeleteItemCommand({
+                TableName: this.tableName,
+                Key: { pk: { S: pk }, sk: { S: this.itemSortKey(address, META_SORT_KEY) } },
+                ReturnValues: "ALL_OLD",
+            }));
+            // Then the chunks of any version (the value's, and any a failed
+            // write left behind), read consistently for the same reason. The
+            // prefix is this entry's own chunk items, so a plain key's delete
+            // leaves the grouped entries sharing its partition alone.
+            await this.deleteItems(await this.queryItems(pk, {
+                prefix: this.itemSortKey(address, CHUNK_SORT_KEY_PREFIX),
+                projection: "#pk, #sk",
+                consistent: true,
+            }));
+            return holdsLiveValue(removed, this.nowSeconds());
+        }
+        finally {
+            // Again once the items are gone: a read in flight meanwhile may
+            // have kept what it found before they were (see LocalWriteLedger).
+            this.memory?.delete(address.memoryKey);
+            this.localWrites.recordKeyWrite(address.memoryKey);
+        }
     }
     /**
      * Drop every entry stored under one `pk`, without knowing which sort keys
      * exist: the invalidation a group of related entries is worth grouping
-     * for. Returns the number of entries removed. In-memory copies held by
-     * OTHER Lambda containers still serve until their own TTL, as they do
+     * for. Returns the number of live values removed. In-memory copies held
+     * by OTHER Lambda containers still serve until their own TTL, as they do
      * after a single-entry delete.
      */
     async deletePartition(partition) {
-        const normalized = this.normalizePartition(partition);
-        const pk = await this.partitionKey(normalized);
+        const normalized = normalizeCachePartition(partition);
+        this.filler.supersedeFillsWithPrefix(cacheMemoryKeyOf(normalized, ""));
         this.forgetPartition(normalized);
-        const items = await this.queryItems(pk, { projection: "#pk, #sk" });
-        await this.deleteItems(items);
-        return items.filter((item) => this.isManifestSortKey(item.sk?.S)).length;
+        try {
+            const pk = await this.partitionKey(normalized);
+            // Read consistently: a replica may not have seen a value or a
+            // fill's lease the table accepted a moment ago, and whatever this
+            // misses outlives the call.
+            const items = await this.queryItems(pk, {
+                projection: "#pk, #sk, #version, #expiresAt",
+                extraNames: { "#version": "version", "#expiresAt": "expiresAt" },
+                consistent: true,
+            });
+            await this.deleteItems(items);
+            const nowSeconds = this.nowSeconds();
+            return items.filter((item) => this.isManifestSortKey(item.sk?.S) && holdsLiveValue(item, nowSeconds)).length;
+        }
+        finally {
+            // Again once the items are gone, as in delete.
+            this.forgetPartition(normalized);
+            this.localWrites.recordPartitionWrite(normalized);
+        }
     }
     /**
      * The live (unexpired) sort keys stored under one `pk`, in table order.
      * Plain-string entries have no sort key, so they never appear here.
-     * Reading a partition whose values are chunked also reads those chunk
-     * items, so grouping very large values makes listing more expensive.
+     * The partition (or prefix range) is read whatever the limit, and reading
+     * a partition whose values are chunked also reads those chunk items, so
+     * grouping very large values makes listing more expensive.
      */
     async listSortKeys(partition, options = {}) {
-        const pk = await this.partitionKey(this.normalizePartition(partition));
-        const prefix = `${SORT_KEY_MARKER}${encodeSortKey(options.prefix ?? "")}`;
+        const pk = await this.partitionKey(normalizeCachePartition(partition));
+        const prefix = `${SORT_KEY_MARKER}${encodeCacheSortKey(options.prefix ?? "")}`;
         const limit = options.limit === undefined ? undefined : assertPositiveInteger(options.limit, "limit");
         const nowSeconds = this.nowSeconds();
-        const items = await this.queryItems(pk, { prefix, projection: "#sk, #expiresAt", extraNames: { "#expiresAt": "expiresAt" } });
+        const items = await this.queryItems(pk, {
+            prefix,
+            projection: "#sk, #expiresAt, #version",
+            extraNames: { "#expiresAt": "expiresAt", "#version": "version" },
+        });
         const sortKeys = [];
         for (const item of items) {
             const sk = item.sk?.S;
             if (!sk || !this.isManifestSortKey(sk))
                 continue;
-            if (Number(item.expiresAt?.N) <= nowSeconds)
+            // A manifest item holding only a fill's lease, or an expired value, has nothing to list.
+            if (!holdsLiveValue(item, nowSeconds))
                 continue;
-            sortKeys.push(decodeSortKey(sk.slice(SORT_KEY_MARKER.length, -(META_SORT_KEY.length + 1))));
+            sortKeys.push(decodeCacheSortKey(sk.slice(SORT_KEY_MARKER.length, -(META_SORT_KEY.length + 1))));
             if (limit !== undefined && sortKeys.length >= limit)
                 break;
         }
         return sortKeys;
     }
-    async getOrSet(key, factory, options = {}) {
-        const address = this.normalizeKey(key);
-        const current = this.inFlight.get(address.memoryKey);
-        if (current)
-            return current;
-        const fill = this.getOrSetFailOpen(address, factory, options).finally(() => {
-            this.inFlight.delete(address.memoryKey);
-        });
-        this.inFlight.set(address.memoryKey, fill);
-        return fill;
-    }
-    /**
-     * Cache infrastructure is best-effort for getOrSet: read, lease, or write
-     * failures return the loader value. Loader failures still propagate and the
-     * loader is never repeated after it has completed successfully.
-     */
-    async getOrSetFailOpen(address, factory, options) {
-        let factoryStarted = false;
-        let factoryCompleted = false;
-        let factoryValue;
-        const trackedFactory = async () => {
-            factoryStarted = true;
-            factoryValue = await factory();
-            factoryCompleted = true;
-            return factoryValue;
-        };
-        try {
+    async getOrSet(key, loader, options = {}) {
+        const address = normalizeCacheKey(key);
+        const settings = resolveGetOrSetOptions(options, this.defaultTtlSeconds);
+        // Cache infrastructure is best-effort here: a read, lease or write
+        // failure returns the loader's value (see LambderCacheFiller).
+        return this.filler.getOrSet(address.memoryKey, loader, async (load) => {
             const existing = await this.getByAddress(address);
             if (existing !== undefined)
                 return existing;
-            return await this.fill(address, trackedFactory, options);
-        }
-        catch (error) {
-            if (factoryStarted && !factoryCompleted)
-                throw error;
-            console.error(`DynamoDB cache failed open in ${this.namespace} for ${address.memoryKey}`, error);
-            if (factoryCompleted)
-                return factoryValue;
-            return trackedFactory();
-        }
+            return await this.fill(address, load, settings);
+        });
     }
-    async fill(address, factory, options) {
-        const leaseSeconds = assertPositiveInteger(options.leaseSeconds ?? 15, "leaseSeconds");
-        const waitForFillMs = assertPositiveInteger(options.waitForFillMs ?? 5_000, "waitForFillMs");
+    /** The fill once a read found nothing: one container loads under the lease while the others wait for its value. */
+    async fill(address, load, { ttlSeconds, leaseSeconds, waitForFillMs }) {
         const pk = await this.partitionKey(address.partition);
         const owner = await randomUUID();
-        if (await this.acquireLease(pk, address, owner, leaseSeconds)) {
+        // The load under the lease just taken, published only while the lease
+        // is still this fill's (see setByAddress). A publish that answered,
+        // landed or refused, leaves no lease of this fill's on the item; any
+        // other end (the loader threw or answered undefined, a write
+        // superseded the fill, the publish failed) releases it.
+        const fillUnderLease = async () => {
+            let published = false;
             try {
-                const value = await factory();
-                await this.setByAddress(address, value, { ttlSeconds: options.ttlSeconds });
-                return value;
+                return await load(async (value) => {
+                    const stored = await this.setByAddress(address, value, ttlSeconds, owner);
+                    published = true;
+                    return stored;
+                });
             }
             finally {
-                await this.releaseLease(pk, address, owner);
+                if (!published)
+                    await this.releaseLease(pk, address, owner);
             }
-        }
+        };
         const deadline = Date.now() + waitForFillMs;
         let delay = 50;
-        while (Date.now() < deadline) {
+        for (;;) {
+            const attempt = await this.takeLease(pk, address, owner, leaseSeconds);
+            if (attempt === "taken")
+                return await fillUnderLease();
+            if (attempt !== "held") {
+                // Refused over a live value: the read that found the entry
+                // missing reached a replica that had not seen it yet. The
+                // refusal carries the manifest as the leader holds it, so it
+                // serves the value, its chunks read consistently when it has
+                // any, rather than waiting out the lag.
+                const value = await this.localWrites.watch(address.memoryKey, (overtaken) => this.readLeaderEntry(pk, address, attempt.filled, overtaken));
+                if (value !== undefined)
+                    return value;
+            }
+            if (Date.now() >= deadline)
+                break;
             await sleep(delay + Math.floor(Math.random() * 25));
             const value = await this.getByAddress(address);
             if (value !== undefined)
                 return value;
-            if (await this.acquireLease(pk, address, owner, leaseSeconds)) {
-                try {
-                    const loaded = await factory();
-                    await this.setByAddress(address, loaded, { ttlSeconds: options.ttlSeconds });
-                    return loaded;
-                }
-                finally {
-                    await this.releaseLease(pk, address, owner);
-                }
-            }
             delay = Math.min(delay * 2, 500);
         }
         throw new Error(`Timed out waiting for DynamoDB cache fill in ${this.namespace}`);
     }
-    async acquireLease(pk, address, owner, leaseSeconds) {
+    /**
+     * Takes the fill lease on one entry by writing it onto the entry's
+     * manifest item, a manifest with no value that the fill's publish then
+     * replaces. Readers find no value on it, so to them it is a miss.
+     *
+     * The write is conditional on the item holding nothing live: none at
+     * all, or one whose expiresAt has passed, whether an expired value or a
+     * lapsed lease. So a lease never hides a live value, and a waiter taking
+     * over a lapsed lease replaces the first holder's, whose publish is then
+     * refused. An expired value the lease replaces goes at once, its chunks
+     * with it, and readers see a miss until the fill publishes, as they did
+     * from the moment it expired. DynamoDB's TTL removes a lease its holder
+     * abandoned.
+     *
+     * A refusal hands back the item that refused it: "held" for another
+     * fill's lease, and for a live value its manifest, which the caller
+     * serves the value from. A manifest this instance cannot read (written
+     * under another chunkBytes or maxValueBytes) is a miss to every read here
+     * and would otherwise block fills until its TTL, so the lease is taken
+     * over it, conditioned on it being the same manifest still.
+     */
+    async takeLease(pk, address, owner, leaseSeconds, unreadableVersion) {
         const now = this.nowSeconds();
+        let current;
         try {
             const { client, sdk } = await this.ready();
-            await client.send(new sdk.PutItemCommand({
+            const response = await client.send(new sdk.PutItemCommand({
                 TableName: this.tableName,
                 Item: {
                     pk: { S: pk },
-                    sk: { S: this.itemSortKey(address, LOCK_SORT_KEY) },
-                    owner: { S: owner },
-                    expiresAt: { N: String(now + leaseSeconds) },
+                    sk: { S: this.itemSortKey(address, META_SORT_KEY) },
+                    leaseOwner: { S: owner },
+                    // The first second the lease no longer holds, as a
+                    // value's expiresAt is the first second it no longer
+                    // reads. Counted from the whole second the lease was
+                    // taken in, so one taken late in a second still lasts
+                    // its full length.
+                    expiresAt: { N: String(now + leaseSeconds + 1) },
                 },
-                ConditionExpression: "attribute_not_exists(#pk) OR #expiresAt < :now",
-                ExpressionAttributeNames: { "#pk": "pk", "#expiresAt": "expiresAt" },
-                ExpressionAttributeValues: { ":now": { N: String(now) } },
+                ...(unreadableVersion === undefined ? {
+                    // An item without expiresAt is none a cache wrote; a
+                    // comparison with a missing attribute is false, so it
+                    // is allowed by name rather than refused for ever.
+                    ConditionExpression: "attribute_not_exists(#expiresAt) OR #expiresAt <= :now",
+                    ExpressionAttributeNames: { "#expiresAt": "expiresAt" },
+                    ExpressionAttributeValues: { ":now": { N: String(now) } },
+                } : {
+                    ConditionExpression: "attribute_not_exists(#leaseOwner) AND #version = :version",
+                    ExpressionAttributeNames: { "#leaseOwner": "leaseOwner", "#version": "version" },
+                    ExpressionAttributeValues: { ":version": { S: unreadableVersion } },
+                }),
+                ReturnValues: "ALL_OLD",
+                ReturnValuesOnConditionCheckFailure: "ALL_OLD",
             }));
-            return true;
+            await this.deleteReplacedChunks(pk, address, response.Attributes);
+            return "taken";
         }
         catch (error) {
-            if (isConditionalCheckFailure(error))
-                return false;
-            throw error;
+            if (!isConditionalCheckFailure(error))
+                throw error;
+            current = error.Item;
         }
-    }
+        const holder = current?.leaseOwner?.S;
+        // Refused over this very lease when the SDK retried an attempt that
+        // had already landed: taken, not refused.
+        if (holder === owner)
+            return "taken";
+        if (!current || holder !== undefined)
+            return "held";
+        const manifest = this.parseManifest(current);
+        if (manifest && manifest.expiresAt > now)
+            return { filled: manifest };
+        const version = current.version?.S;
+        if (unreadableVersion !== undefined || version === undefined)
+            return "held";
+        return await this.takeLease(pk, address, owner, leaseSeconds, version);
+    }
+    /** Clears this fill's lease off the manifest item, conditional on it still being this fill's, so it never touches what replaced it. */
     async releaseLease(pk, address, owner) {
         try {
             const { client, sdk } = await this.ready();
             await client.send(new sdk.DeleteItemCommand({
                 TableName: this.tableName,
-                Key: { pk: { S: pk }, sk: { S: this.itemSortKey(address, LOCK_SORT_KEY) } },
-                ConditionExpression: "#owner = :owner",
-                ExpressionAttributeNames: { "#owner": "owner" },
+                Key: { pk: { S: pk }, sk: { S: this.itemSortKey(address, META_SORT_KEY) } },
+                ConditionExpression: "#leaseOwner = :owner",
+                ExpressionAttributeNames: { "#leaseOwner": "leaseOwner" },
                 ExpressionAttributeValues: { ":owner": { S: owner } },
             }));
         }
@@ -418,16 +739,21 @@ export class LambderDdbCache {
             }
         }
     }
-    async readManifest(pk, address) {
+    async readManifest(pk, address, consistent) {
         const { client, sdk } = await this.ready();
         const response = await client.send(new sdk.GetItemCommand({
             TableName: this.tableName,
             Key: { pk: { S: pk }, sk: { S: this.itemSortKey(address, META_SORT_KEY) } },
-            ConsistentRead: false,
+            ConsistentRead: consistent,
         }));
-        const item = response.Item;
-        if (!item)
-            return undefined;
+        return response.Item ? this.parseManifest(response.Item) : undefined;
+    }
+    /**
+     * The value a manifest item describes, or undefined when it describes
+     * none this instance can read: a fill's lease with no value under it
+     * yet, or a manifest that does not add up. Expiry is the caller's check.
+     */
+    parseManifest(item) {
         const version = item.version?.S;
         const encoding = item.encoding?.S;
         const chunkCount = Number(item.chunkCount?.N);
@@ -470,12 +796,13 @@ export class LambderDdbCache {
             inlineData,
         };
     }
-    async readChunks(pk, address, manifest) {
+    async readChunks(pk, address, manifest, consistent) {
         const prefix = this.itemSortKey(address, `${CHUNK_SORT_KEY_PREFIX}${manifest.version}#`);
         const items = await this.queryItems(pk, {
             prefix,
             projection: "#sk, #data",
             extraNames: { "#data": "data" },
+            consistent,
         });
         const chunks = items
             .filter((item) => item.sk?.S && item.data?.B)
@@ -509,7 +836,7 @@ export class LambderDdbCache {
                 },
                 ProjectionExpression: options.projection,
                 ExclusiveStartKey: cursor,
-                ConsistentRead: false,
+                ConsistentRead: options.consistent ?? false,
             }));
             items.push(...(response.Items ?? []));
             cursor = response.LastEvaluatedKey;
@@ -564,52 +891,28 @@ export class LambderDdbCache {
         const ttl = expiresAt * 1000 - this.now();
         if (ttl <= 0)
             return;
-        this.memory.set(key, { stored, encoding, uncompressedBytes, expiresAt }, { ttl });
-    }
-    normalizeKey(key) {
-        if (typeof key === "string") {
-            const partition = this.normalizePartition(key);
-            return { partition, sortKey: null, memoryKey: this.memoryKeyOf(partition, null) };
-        }
-        if (!key || typeof key !== "object")
-            throw new Error("Cache key is required");
-        const partition = this.normalizePartition(key.pk);
-        const sortKey = key.sk;
-        if (typeof sortKey !== "string" || !sortKey.trim())
-            throw new Error("Cache sort key is required");
-        const encodedBytes = Buffer.byteLength(encodeSortKey(sortKey), "utf8");
-        if (encodedBytes > MAX_SORT_KEY_BYTES) {
-            throw new Error(`Cache sort key must be at most ${MAX_SORT_KEY_BYTES} UTF-8 bytes once escaped (${encodedBytes})`);
+        // Kept for as long as the entry lives, and counted by its length, so
+        // it must own exactly those bytes. A small Buffer is often a view onto
+        // something larger (a slice of zlib's 16 KB output chunk, a copy in
+        // Node's shared 8 KB pool), and kept as it is it would pin all of it.
+        let owned = stored;
+        if (stored.byteLength !== stored.buffer.byteLength) {
+            owned = Buffer.allocUnsafeSlow(stored.byteLength);
+            stored.copy(owned);
         }
-        return { partition, sortKey, memoryKey: this.memoryKeyOf(partition, sortKey) };
-    }
-    normalizePartition(key) {
-        if (typeof key !== "string" || !key.trim())
-            throw new Error("Cache key is required");
-        if (Buffer.byteLength(key, "utf8") > 8 * 1024) {
-            throw new Error("Cache key must be at most 8192 UTF-8 bytes");
-        }
-        return key;
-    }
-    /** Length-prefixed so a partition ending in the separator cannot collide with a sort key. */
-    memoryKeyOf(partition, sortKey) {
-        return `${partition.length}:${partition}#${sortKey ?? ""}`;
+        this.memory.set(key, { stored: owned, encoding, uncompressedBytes, expiresAt }, { ttl });
     }
     async partitionKey(key) {
         return `${this.keyPrefix}#${this.namespace}#${await sha256(key)}`;
     }
     /**
-     * One of an entry's item keys. A plain-string entry keeps the bare
-     * suffix it has always used; a grouped one nests under its escaped sort
-     * key, whose trailing `#` is an unambiguous boundary because an escaped
-     * sort key never contains a bare `#`.
+     * One of an entry's item keys. A plain-string entry uses the bare
+     * suffix; a grouped one nests under its escaped sort key, whose trailing
+     * `#` is an unambiguous boundary because an escaped sort key never
+     * contains a bare `#`.
      */
     itemSortKey(address, suffix) {
-        return address.sortKey === null ? suffix : `${SORT_KEY_MARKER}${encodeSortKey(address.sortKey)}#${suffix}`;
-    }
-    /** The prefix covering every item of a grouped entry; null for a plain-string entry, which owns the bare item keys instead. */
-    entryItemPrefix(address) {
-        return address.sortKey === null ? null : `${SORT_KEY_MARKER}${encodeSortKey(address.sortKey)}#`;
+        return address.sortKey === null ? suffix : `${SORT_KEY_MARKER}${encodeCacheSortKey(address.sortKey)}#${suffix}`;
     }
     isManifestSortKey(sk) {
         return sk === META_SORT_KEY || (!!sk && sk.startsWith(SORT_KEY_MARKER) && sk.endsWith(`#${META_SORT_KEY}`));
@@ -621,7 +924,7 @@ export class LambderDdbCache {
     forgetPartition(partition) {
         if (!this.memory)
             return;
-        const prefix = this.memoryKeyOf(partition, "");
+        const prefix = cacheMemoryKeyOf(partition, "");
         for (const key of [...this.memory.keys()]) {
             if (key.startsWith(prefix))
                 this.memory.delete(key);