@minnowdb/core 0.9.0 → 0.10.0

package/dist/storage/opfs/store.js

@@ -3,11 +3,15 @@ import { validateTempRunPage, validateTempRunPageIdentity } from "../toolkit/rec
 import { OpfsTree, encodeSegment, isDomError } from "./files.js";
 import { LOG_FORMAT_VERSION } from "../toolkit/wire.js";
 import { OpfsLeader } from "./leader.js";
-import { rehydrateStoreError, estimateRpcValueBytes, fingerprintStoreRequest, MAX_OPFS_RPC_MESSAGE_BYTES, parseStoreRpcMessage, serializeStoreError } from "./rpc.js";
+import { rehydrateStoreError, estimateRpcValueBytes, fingerprintStoreRequest, MAX_OPFS_RPC_HOLD_MS, MAX_OPFS_RPC_MESSAGE_BYTES, parseStoreRpcMessage, serializeStoreError } from "./rpc.js";
 const RPC_TIMEOUT_MS = 1e3;
 const DISCOVERY_WAIT_MS = 150;
 const DISPATCH_ATTEMPTS = 10;
 const YIELD_COOLDOWN_MS = 3e3;
+const HANDOVER_GRACE_MS = 1500;
+const HIDDEN_IDLE_RELEASE_MS = 15e3;
+const MUTATION_PATIENCE_ROUNDS = 3;
+const DECLINE_AFTER_CLOSE_MS = 1e3;
 const DEDUPE_CACHE_SIZE = 512;
 const RPC_IN_FLIGHT_LIMIT = 512;
 const RPC_IN_FLIGHT_MUTATION_BYTES = 128 * 1024 * 1024;
@@ -175,7 +179,13 @@ class OpfsBlockStore {
   #durability;
   #checkpointEntries;
   #cleanupLimitBytes;
+  #servedLedgerAgeMs;
+  #servedLedgerResultBytes;
   #rpcTimeoutMs;
+  #yieldCooldownMs;
+  #hiddenIdleReleaseMs;
+  #handoverGraceMs;
+  #onDiagnostic;
   #instanceId = crypto.randomUUID();
   #channelName;
   #channel;
@@ -201,12 +211,26 @@ class OpfsBlockStore {
   #servedMutationGateForTests;
   #dropNextRpcResultForTests = false;
   #reacquireTimer;
+  #served = /* @__PURE__ */ new Map();
+  #keepaliveTimer;
+  #deferredBid;
+  #hiddenIdleTimer;
+  #inboxLingerTimer;
+  #lastActivityAt = Date.now();
+  #othersSeen = false;
+  #gracefulLeaders = /* @__PURE__ */ new Set();
   constructor(tree, options) {
     this.#tree = tree;
     this.#durability = options.durability ?? "strict";
     this.#checkpointEntries = options.checkpointEntries;
     this.#cleanupLimitBytes = options.cleanupLimitBytes;
+    this.#servedLedgerAgeMs = options.servedLedgerAgeMs;
+    this.#servedLedgerResultBytes = options.servedLedgerResultBytes;
     this.#rpcTimeoutMs = options.rpcTimeoutMs ?? RPC_TIMEOUT_MS;
+    this.#yieldCooldownMs = options.yieldCooldownMs ?? YIELD_COOLDOWN_MS;
+    this.#hiddenIdleReleaseMs = options.hiddenIdleReleaseMs ?? HIDDEN_IDLE_RELEASE_MS;
+    this.#handoverGraceMs = options.handoverGraceMs ?? HANDOVER_GRACE_MS;
+    this.#onDiagnostic = options.onDiagnostic;
     this.#channelName = `minnowdb-store:${options.name}`;
     this.liveQueryChannelName = `minnowdb-live:opfs:${options.name}`;
   }
@@ -246,11 +270,26 @@ class OpfsBlockStore {
       throw error;
     }
   }
-  async #tryBecomeLeader() {
+  #diagnostic(error, context) {
+    try {
+      this.#onDiagnostic?.(error, context);
+    } catch {
+    }
+  }
+  #electInBackground(context, force = false) {
+    this.#tryBecomeLeader(force).catch((error) => {
+      this.#diagnostic(error, context);
+    });
+  }
+  async #tryBecomeLeader(force = false) {
     if (this.#closed)
       return false;
     if (this.#leader !== void 0)
       return true;
+    if (this.#yielding !== void 0)
+      return false;
+    if (!force && Date.now() - this.#lastYieldAt < this.#handoverGraceMs)
+      return false;
     if (this.#electing !== void 0)
       return this.#electing;
     const election = this.#elect();
@@ -275,7 +314,7 @@ class OpfsBlockStore {
     try {
       slotA = await this.#openWithRetry(["checkpoint-a"]);
       slotB = await this.#openWithRetry(["checkpoint-b"]);
-      this.#leader = await OpfsLeader.recover(this.#tree, this.#durability === "strict", { wal, slotA, slotB }, this.#checkpointEntries, this.#cleanupLimitBytes);
+      this.#leader = await OpfsLeader.recover(this.#tree, this.#durability === "strict", { wal, slotA, slotB }, this.#checkpointEntries, this.#cleanupLimitBytes, this.#onDiagnostic, this.#servedLedgerAgeMs, this.#servedLedgerResultBytes);
     } catch (error) {
       wal.close();
       slotA?.close();
@@ -287,15 +326,102 @@ class OpfsBlockStore {
     if (this.#closed) {
       const leader = this.#leader;
       this.#leader = void 0;
-      await leader.shutdown().catch(() => {
+      await leader.shutdown().catch((error) => {
+        this.#diagnostic(error, "opfs shutdown after close during election");
         leader.crash();
       });
       return false;
     }
     this.#knownLeader = this.#instanceId;
+    this.#leader.onBeforeCheckpoint = (expectedMs) => {
+      this.#holdServed(expectedMs);
+    };
     this.#post({ kind: "leader", leaderId: this.#instanceId });
+    this.#post({ kind: "state", leaderId: this.#instanceId, foreground: this.#foreground });
+    if (!this.#foreground)
+      this.#armHiddenIdleTimer();
     return true;
   }
+  #holdServed(expectedMs) {
+    if (this.#served.size === 0)
+      return;
+    const ms = Math.min(MAX_OPFS_RPC_HOLD_MS, expectedMs * 2 + this.#rpcTimeoutMs);
+    for (const [requestId, from] of this.#served)
+      this.#answer(from, { kind: "hold", requestId, ms });
+  }
+  #keepaliveSuppressedForTests = false;
+  _suppressKeepaliveForTests() {
+    this.#keepaliveSuppressedForTests = true;
+    this.#stopKeepalive();
+  }
+  #startKeepalive() {
+    if (this.#keepaliveTimer !== void 0 || this.#keepaliveSuppressedForTests)
+      return;
+    const timer = setInterval(() => {
+      if (this.#served.size === 0) {
+        clearInterval(timer);
+        if (this.#keepaliveTimer === timer)
+          this.#keepaliveTimer = void 0;
+        return;
+      }
+      for (const [requestId, from] of this.#served) {
+        this.#answer(from, { kind: "busy", requestId });
+      }
+    }, Math.max(1, Math.floor(this.#rpcTimeoutMs / 2)));
+    timer.unref?.();
+    this.#keepaliveTimer = timer;
+  }
+  #stopKeepalive() {
+    if (this.#keepaliveTimer === void 0)
+      return;
+    clearInterval(this.#keepaliveTimer);
+    this.#keepaliveTimer = void 0;
+  }
+  #armHiddenIdleTimer() {
+    this.#clearHiddenIdleTimer();
+    if (this.#foreground || this.#leader === void 0 || this.#closed)
+      return;
+    const elapsed = Date.now() - this.#lastActivityAt;
+    const timer = setTimeout(() => {
+      this.#hiddenIdleTimer = void 0;
+      if (this.#closed || this.#leader === void 0 || this.#foreground)
+        return;
+      if (this.#othersSeen && this.#served.size === 0 && this.#yielding === void 0 && Date.now() - this.#lastActivityAt >= this.#hiddenIdleReleaseMs) {
+        this.#demote().catch((error) => {
+          this.#diagnostic(error, "opfs idle release");
+        });
+        return;
+      }
+      this.#armHiddenIdleTimer();
+    }, Math.max(0, this.#hiddenIdleReleaseMs - elapsed));
+    timer.unref?.();
+    this.#hiddenIdleTimer = timer;
+  }
+  #clearHiddenIdleTimer() {
+    if (this.#hiddenIdleTimer === void 0)
+      return;
+    clearTimeout(this.#hiddenIdleTimer);
+    this.#hiddenIdleTimer = void 0;
+  }
+  async #demote() {
+    const leader = this.#leader;
+    if (leader === void 0)
+      return;
+    this.#leader = void 0;
+    this.#knownLeader = void 0;
+    const shutdown = leader.shutdown().catch((error) => {
+      this.#diagnostic(error, "opfs idle release shutdown");
+      leader.crash();
+    });
+    this.#yielding = shutdown;
+    await shutdown;
+    if (this.#yielding === shutdown)
+      this.#yielding = void 0;
+    if (this.#closed)
+      return;
+    this.#closeAnswerChannels();
+    this.#post({ kind: "released", leaderId: this.#instanceId });
+  }
   async #openWithRetry(path) {
     for (let attempt = 0; ; attempt += 1) {
       try {
@@ -310,19 +436,66 @@ class OpfsBlockStore {
     }
   }
   setForeground(foreground) {
+    if (this.#closed)
+      return;
+    const changed = this.#foreground !== foreground;
     this.#foreground = foreground;
-    if (foreground && this.#leader === void 0 && !this.#closed) {
+    if (this.#leader !== void 0) {
+      if (foreground) {
+        this.#clearHiddenIdleTimer();
+        this.#clearDeferredBid();
+      } else {
+        this.#lastActivityAt = Date.now();
+        this.#armHiddenIdleTimer();
+      }
+      if (changed)
+        this.#post({ kind: "state", leaderId: this.#instanceId, foreground });
+      return;
+    }
+    if (foreground)
       this.#post({ kind: "bid", bidderId: this.#instanceId, foreground: true });
+  }
+  #clearDeferredBid() {
+    if (this.#deferredBid === void 0)
+      return;
+    clearTimeout(this.#deferredBid.timer);
+    this.#deferredBid = void 0;
+  }
+  #considerBid(bidderId) {
+    if (this.#leader === void 0 || this.#foreground || this.#yielding !== void 0)
+      return;
+    const remaining = this.#yieldCooldownMs - (Date.now() - this.#lastYieldAt);
+    if (remaining <= 0) {
+      this.#clearDeferredBid();
+      this.#yieldLeadership(bidderId).catch((error) => {
+        this.#diagnostic(error, "opfs yield");
+      });
+      return;
+    }
+    if (this.#deferredBid !== void 0) {
+      this.#deferredBid.bidderId = bidderId;
+      return;
     }
+    const timer = setTimeout(() => {
+      const deferred = this.#deferredBid;
+      this.#deferredBid = void 0;
+      if (deferred !== void 0 && !this.#closed)
+        this.#considerBid(deferred.bidderId);
+    }, remaining);
+    timer.unref?.();
+    this.#deferredBid = { bidderId, timer };
   }
   async #yieldLeadership(to) {
     const leader = this.#leader;
     if (leader === void 0)
       return;
+    this.#clearHiddenIdleTimer();
+    this.#clearDeferredBid();
     this.#leader = void 0;
     this.#knownLeader = void 0;
     this.#lastYieldAt = Date.now();
-    const shutdown = leader.shutdown().catch(() => {
+    const shutdown = leader.shutdown().catch((error) => {
+      this.#diagnostic(error, "opfs yield shutdown");
       leader.crash();
     });
     this.#yielding = shutdown;
@@ -333,22 +506,32 @@ class OpfsBlockStore {
       return;
     this.#closeAnswerChannels();
     this.#post({ kind: "yield", to });
+    this.#post({ kind: "released", leaderId: this.#instanceId });
     if (this.#reacquireTimer !== void 0)
       clearTimeout(this.#reacquireTimer);
     this.#reacquireTimer = setTimeout(() => {
-      if (this.#knownLeader === void 0 && !this.#closed)
-        void this.#tryBecomeLeader();
-    }, 1500);
+      if (this.#knownLeader === void 0 && !this.#closed) {
+        this.#electInBackground("opfs reacquire after yield", true);
+      }
+    }, this.#handoverGraceMs);
   }
   #yielding;
   #onMessage(message) {
-    if (this.#closed)
+    if (this.#closed) {
+      if (message.kind === "op")
+        this.#decline(message);
       return;
+    }
     if (this.#coordinationPausedForTests)
       return;
+    this.#othersSeen = true;
     switch (message.kind) {
       case "op": {
-        if (this.#leader !== void 0) {
+        if (this.#leader === void 0) {
+          this.#decline(message);
+          return;
+        }
+        {
           const requestBytes = estimateRpcValueBytes(message.args);
           if (this.#servedRequestCount >= RPC_SERVER_ADMISSION_LIMIT || this.#servedRequestBytes + requestBytes > RPC_SERVER_ADMISSION_BYTES) {
             this.#answer(message.from, {
@@ -361,13 +544,51 @@ class OpfsBlockStore {
           }
           this.#servedRequestCount += 1;
           this.#servedRequestBytes += requestBytes;
-          void this.#serveOp(message).finally(() => {
+          this.#lastActivityAt = Date.now();
+          const alreadyServing = this.#served.has(message.requestId);
+          this.#served.set(message.requestId, message.from);
+          this.#startKeepalive();
+          void this.#serveOp(message).catch((error) => {
+            this.#diagnostic(error, `opfs served ${message.method}`);
+            this.#answer(message.from, {
+              kind: "result",
+              requestId: message.requestId,
+              ok: false,
+              error: serializeStoreError(error)
+            });
+          }).finally(() => {
             this.#servedRequestCount = Math.max(0, this.#servedRequestCount - 1);
             this.#servedRequestBytes = Math.max(0, this.#servedRequestBytes - requestBytes);
+            if (!alreadyServing)
+              this.#served.delete(message.requestId);
+            this.#lastActivityAt = Date.now();
           });
         }
         return;
       }
+      case "declined": {
+        const pending = this.#takePending(message.requestId);
+        if (pending === void 0)
+          return;
+        clearTimeout(pending.timer);
+        pending.reject(RPC_DECLINED);
+        return;
+      }
+      case "uncertain": {
+        const pending = this.#takePending(message.requestId);
+        if (pending === void 0)
+          return;
+        clearTimeout(pending.timer);
+        pending.reject(new OpfsUncertainOutcomeError(pending.message.method));
+        return;
+      }
+      case "hold": {
+        const pending = this.#pending.get(message.requestId);
+        if (pending === void 0)
+          return;
+        this.#armPendingTimer(pending, Math.max(message.ms, this.#rpcTimeoutMs));
+        return;
+      }
       case "result": {
         const pending = this.#takePending(message.requestId);
         if (pending === void 0)
@@ -383,26 +604,28 @@ class OpfsBlockStore {
         const pending = this.#pending.get(message.requestId);
         if (pending === void 0)
           return;
-        clearTimeout(pending.timer);
-        pending.timer = setTimeout(() => {
-          const expired = this.#takePending(message.requestId);
-          expired?.reject(RPC_TIMED_OUT);
-        }, this.#rpcTimeoutMs);
+        this.#armPendingTimer(pending, this.#rpcTimeoutMs);
         return;
       }
       case "leader": {
         this.#knownLeader = message.leaderId;
+        this.#gracefulLeaders.delete(message.leaderId);
         if (this.#reacquireTimer !== void 0) {
           clearTimeout(this.#reacquireTimer);
           this.#reacquireTimer = void 0;
         }
-        for (const [requestId, pending] of this.#pending) {
-          if (pending.sentTo === message.leaderId)
+        for (const pending of this.#pending.values()) {
+          if (pending.sentTo === message.leaderId) {
+            if (pending.awaitingPing)
+              this.#armPendingTimer(pending, this.#rpcTimeoutMs);
             continue;
-          if (!READ_METHODS.has(pending.message.method)) {
-            this.#takePending(requestId);
-            clearTimeout(pending.timer);
-            pending.reject(new OpfsUncertainOutcomeError(pending.message.method));
+          }
+          if (pending.sentTo === void 0) {
+            pending.sentTo = message.leaderId;
+            this.#send(message.leaderId, pending.message);
+            continue;
+          }
+          if (!READ_METHODS.has(pending.message.method) && this.#gracefulLeaders.has(pending.sentTo)) {
             continue;
           }
           pending.sentTo = message.leaderId;
@@ -413,6 +636,12 @@ class OpfsBlockStore {
         }
         return;
       }
+      case "state": {
+        if (!message.foreground && this.#foreground && this.#leader === void 0 && message.leaderId !== this.#instanceId) {
+          this.#post({ kind: "bid", bidderId: this.#instanceId, foreground: true });
+        }
+        return;
+      }
       case "ping": {
         if (this.#leader !== void 0) {
           this.#post({ kind: "leader", leaderId: this.#instanceId });
@@ -420,23 +649,61 @@ class OpfsBlockStore {
         return;
       }
       case "bid": {
-        if (this.#leader !== void 0 && !this.#foreground && message.foreground && message.bidderId !== this.#instanceId && Date.now() - this.#lastYieldAt > YIELD_COOLDOWN_MS) {
-          void this.#yieldLeadership(message.bidderId);
+        if (message.foreground && message.bidderId !== this.#instanceId) {
+          this.#considerBid(message.bidderId);
         }
         return;
       }
       case "yield": {
-        if (message.to === this.#instanceId)
-          void this.#tryBecomeLeader();
+        if (message.to === this.#instanceId) {
+          this.#knownLeader = void 0;
+          this.#electInBackground("opfs election after yield", true);
+        }
         return;
       }
       case "released": {
         if (this.#knownLeader === message.leaderId)
           this.#knownLeader = void 0;
+        this.#gracefulLeaders.add(message.leaderId);
+        if (this.#gracefulLeaders.size > 16) {
+          const [oldest] = this.#gracefulLeaders;
+          if (oldest !== void 0)
+            this.#gracefulLeaders.delete(oldest);
+        }
         return;
       }
     }
   }
+  #decline(message) {
+    this.#postOnce(message.from, {
+      kind: "declined",
+      requestId: message.requestId,
+      reason: "not-leader"
+    });
+  }
+  #armPendingTimer(pending, ms) {
+    clearTimeout(pending.timer);
+    pending.awaitingPing = false;
+    pending.timer = setTimeout(() => {
+      this.#onPendingTimeout(pending.message.requestId);
+    }, ms);
+  }
+  #onPendingTimeout(requestId) {
+    const pending = this.#pending.get(requestId);
+    if (pending === void 0)
+      return;
+    if (READ_METHODS.has(pending.message.method) || pending.awaitingPing || pending.patienceRounds >= MUTATION_PATIENCE_ROUNDS || this.#channel === void 0) {
+      this.#takePending(requestId);
+      pending.reject(RPC_TIMED_OUT);
+      return;
+    }
+    pending.patienceRounds += 1;
+    pending.awaitingPing = true;
+    this.#post({ kind: "ping" });
+    pending.timer = setTimeout(() => {
+      this.#onPendingTimeout(requestId);
+    }, DISCOVERY_WAIT_MS);
+  }
   #takePending(requestId) {
     const pending = this.#pending.get(requestId);
     if (pending === void 0)
@@ -446,7 +713,7 @@ class OpfsBlockStore {
     return pending;
   }
   async #serveOp(message) {
-    const requestKey = `${String(message.from.length)}:${message.from}${message.requestId}`;
+    const requestKey = servedRequestKey(message.from, message.requestId);
     if (this.#inFlightMutations.has(requestKey) || this.#settledMutations.has(requestKey)) {
       await this.#serveOpLocked(message, requestKey);
       return;
@@ -476,10 +743,38 @@ class OpfsBlockStore {
     let fingerprint;
     try {
       fingerprint = await fingerprintStoreRequest(message.method, message.args);
-    } catch {
+    } catch (error) {
+      this.#answer(message.from, {
+        kind: "result",
+        requestId: message.requestId,
+        ok: false,
+        error: serializeStoreError(error)
+      });
       return;
     }
-    const remembered = isRead ? void 0 : this.#settledMutations.get(requestKey);
+    let remembered = isRead ? void 0 : this.#settledMutations.get(requestKey);
+    const inFlight = isRead || remembered !== void 0 ? void 0 : this.#inFlightMutations.get(requestKey);
+    if (remembered === void 0 && inFlight === void 0 && !isRead) {
+      const logged = leader.servedOutcome(requestKey);
+      if (logged !== void 0) {
+        if (logged.method !== message.method || logged.signature !== fingerprint.signature) {
+          this.#rejectReusedRequestIdentity(message);
+          return;
+        }
+        if (!logged.settled || logged.withheld === true) {
+          this.#answer(message.from, { kind: "uncertain", requestId: message.requestId });
+          return;
+        }
+        const outcome = { ok: true, value: logged.result };
+        this.#rememberSettledMutation(requestKey, logged.method, { signature: logged.signature, retainedBytes: logged.requestBytes }, outcome);
+        remembered = this.#settledMutations.get(requestKey) ?? {
+          method: logged.method,
+          signature: logged.signature,
+          requestBytes: logged.requestBytes,
+          outcome
+        };
+      }
+    }
     let settled;
     if (remembered !== void 0) {
       if (!sameServedRequest(remembered, message, fingerprint.signature)) {
@@ -489,10 +784,7 @@ class OpfsBlockStore {
       this.#settledMutations.delete(requestKey);
       this.#settledMutations.set(requestKey, remembered);
       settled = remembered.outcome;
-    } else if (!isRead && this.#inFlightMutations.has(requestKey)) {
-      const inFlight = this.#inFlightMutations.get(requestKey);
-      if (inFlight === void 0)
-        throw new Error("In-flight RPC identity disappeared");
+    } else if (inFlight !== void 0) {
       if (!sameServedRequest(inFlight, message, fingerprint.signature)) {
         this.#rejectReusedRequestIdentity(message);
         return;
@@ -500,6 +792,10 @@ class OpfsBlockStore {
       this.#answer(message.from, { kind: "busy", requestId: message.requestId });
       settled = await inFlight.outcome;
     } else {
+      if (!isRead && message.sentAt < leader.servedCoverageSince) {
+        this.#answer(message.from, { kind: "uncertain", requestId: message.requestId });
+        return;
+      }
       if (!isRead && (this.#inFlightMutations.size >= RPC_IN_FLIGHT_LIMIT || this.#inFlightMutationBytes + fingerprint.retainedBytes > RPC_IN_FLIGHT_MUTATION_BYTES)) {
         this.#answer(message.from, {
           kind: "result",
@@ -521,11 +817,15 @@ class OpfsBlockStore {
             this.#readCapacityChanged = { promise, resolve };
           }
           await this.#readCapacityChanged.promise;
-          if (this.#closed || this.#leader !== leader)
+          if (this.#closed || this.#leader !== leader) {
+            this.#decline(message);
             return;
+          }
         }
-        if (this.#closed || this.#leader !== leader)
+        if (this.#closed || this.#leader !== leader) {
+          this.#decline(message);
           return;
+        }
         this.#inFlightReadBytes += reservation;
         try {
           settled = await this.#executeServedOpAfterGate(leader, message, true);
@@ -535,11 +835,17 @@ class OpfsBlockStore {
         }
       } else {
         this.#inFlightMutationBytes += fingerprint.retainedBytes;
-        const execution = this.#executeServedOpAfterGate(leader, message, false);
+        const execution = this.#executeServedOpAfterGate(leader, message, false, {
+          key: requestKey,
+          method: message.method,
+          signature: fingerprint.signature,
+          requestBytes: fingerprint.retainedBytes,
+          sentAt: Math.min(message.sentAt, Date.now())
+        });
         const outcome = execution.then((result) => {
           this.#inFlightMutations.delete(requestKey);
           this.#inFlightMutationBytes = Math.max(0, this.#inFlightMutationBytes - fingerprint.retainedBytes);
-          if (!this.#closed) {
+          if (!this.#closed && result !== DECLINED_OUTCOME) {
             this.#rememberSettledMutation(requestKey, message.method, fingerprint, result);
           }
           return result;
@@ -553,6 +859,8 @@ class OpfsBlockStore {
         settled = await outcome;
       }
     }
+    if (settled === DECLINED_OUTCOME)
+      return;
     if (this.#dropNextRpcResultForTests) {
       this.#dropNextRpcResultForTests = false;
       return;
@@ -601,20 +909,58 @@ class OpfsBlockStore {
     this.#readCapacityChanged = void 0;
     waiting?.resolve();
   }
-  async #executeServedOpAfterGate(leader, message, isRead) {
+  async #executeServedOpAfterGate(leader, message, isRead, request) {
     const gate = this.#servedMutationGateForTests;
     if (!isRead && gate !== void 0)
       await gate;
-    return this.#executeServedOp(leader, message);
+    return this.#executeServedOp(leader, message, request);
+  }
+  #leads(leader) {
+    return this.#leader === leader;
+  }
+  #mutationTail = Promise.resolve();
+  #withMutationTurn(run) {
+    const turn = this.#mutationTail.then(run);
+    this.#mutationTail = turn.then(() => void 0, () => void 0);
+    return turn;
   }
-  async #executeServedOp(leader, message) {
+  async #executeServedOp(leader, message, request) {
+    if (leader.isClosed() || !this.#leads(leader)) {
+      this.#decline(message);
+      return DECLINED_OUTCOME;
+    }
     try {
       const method = leader[message.method];
       if (method === void 0) {
         return { ok: false, error: { name: "Error", message: "Unknown store operation" } };
       }
-      return { ok: true, value: await method.apply(leader, message.args) };
+      if (request === void 0) {
+        return { ok: true, value: await method.apply(leader, message.args) };
+      }
+      return await this.#withMutationTurn(async () => {
+        if (leader.isClosed() || !this.#leads(leader)) {
+          this.#decline(message);
+          return DECLINED_OUTCOME;
+        }
+        leader.servingRequest = request;
+        try {
+          const value = await method.apply(leader, message.args);
+          if (leader.servingRequest !== request) {
+            await leader.completeServed(request.key, value).catch((error) => {
+              this.#diagnostic(error, `opfs served result for ${message.method}`);
+            });
+          }
+          return { ok: true, value };
+        } finally {
+          if (leader.servingRequest === request)
+            leader.servingRequest = void 0;
+        }
+      });
     } catch (error) {
+      if (leader.isClosed() && !this.#leads(leader)) {
+        this.#decline(message);
+        return DECLINED_OUTCOME;
+      }
       return { ok: false, error: serializeStoreError(error) };
     }
   }
@@ -663,9 +1009,20 @@ class OpfsBlockStore {
   #closeChannels() {
     this.#channel?.close();
     this.#channel = void 0;
-    this.#inbox?.close();
-    this.#inbox = void 0;
     this.#closeAnswerChannels();
+    const inbox = this.#inbox;
+    this.#inbox = void 0;
+    if (inbox === void 0)
+      return;
+    if (this.#inboxLingerTimer !== void 0)
+      clearTimeout(this.#inboxLingerTimer);
+    const timer = setTimeout(() => {
+      this.#inboxLingerTimer = void 0;
+      inbox.close();
+    }, DECLINE_AFTER_CLOSE_MS);
+    timer.unref?.();
+    inbox.unref?.();
+    this.#inboxLingerTimer = timer;
   }
   #assertOpen() {
     if (this.#closed)
@@ -674,6 +1031,11 @@ class OpfsBlockStore {
   async #dispatch(method, args) {
     this.#assertOpen();
     const requestId = crypto.randomUUID();
+    const sentAt = Date.now();
+    const isRead = READ_METHODS.has(method);
+    let sentRemotely = false;
+    let mayHaveRun = false;
+    this.#lastActivityAt = sentAt;
     for (let attempt = 0; attempt < DISPATCH_ATTEMPTS; attempt += 1) {
       this.#assertOpen();
       const leader = this.#leader;
@@ -681,7 +1043,25 @@ class OpfsBlockStore {
         const bound = leader[method];
         if (bound === void 0)
           throw new Error(`Unknown store operation: ${method}`);
-        return bound.apply(leader, args);
+        try {
+          if (isRead)
+            return await bound.apply(leader, args);
+          if (sentRemotely) {
+            const key = servedRequestKey(this.#instanceId, requestId);
+            const logged = leader.servedOutcome(key);
+            if (logged !== void 0) {
+              if (!logged.settled || logged.withheld === true) {
+                throw new OpfsUncertainOutcomeError(method);
+              }
+              return logged.result;
+            }
+            if (sentAt < leader.servedCoverageSince)
+              throw new OpfsUncertainOutcomeError(method);
+          }
+          return await this.#withMutationTurn(() => bound.apply(leader, args));
+        } finally {
+          this.#lastActivityAt = Date.now();
+        }
       }
       if (this.#channel === void 0) {
         if (await this.#tryBecomeLeader())
@@ -699,23 +1079,26 @@ class OpfsBlockStore {
         }
       }
       try {
-        return await this.#rpc(requestId, method, args);
+        sentRemotely = true;
+        return await this.#rpc(requestId, method, args, sentAt);
       } catch (error) {
-        if (error === RPC_TIMED_OUT) {
+        if (error === RPC_DECLINED || error === RPC_TIMED_OUT) {
+          if (error === RPC_TIMED_OUT)
+            mayHaveRun = true;
           this.#knownLeader = void 0;
-          if (!READ_METHODS.has(method))
-            throw new OpfsUncertainOutcomeError(method);
           continue;
         }
         throw error;
       }
     }
+    if (!isRead && mayHaveRun)
+      throw new OpfsUncertainOutcomeError(method);
     throw new OpfsCoordinationError("leader-unavailable", method);
   }
   #leaderKnown() {
     return this.#knownLeader !== void 0;
   }
-  #rpc(requestId, method, args) {
+  #rpc(requestId, method, args, sentAt) {
     let retainedBytes;
     try {
       retainedBytes = estimateRpcValueBytes(args);
@@ -726,10 +1109,16 @@ class OpfsBlockStore {
       return Promise.reject(new OpfsCoordinationError("follower-queue-full", method));
     }
     return new Promise((resolve, reject) => {
-      const message = { kind: "op", requestId, from: this.#instanceId, method, args };
+      const message = {
+        kind: "op",
+        requestId,
+        from: this.#instanceId,
+        method,
+        args,
+        sentAt
+      };
       const timer = setTimeout(() => {
-        const expired = this.#takePending(requestId);
-        expired?.reject(RPC_TIMED_OUT);
+        this.#onPendingTimeout(requestId);
       }, this.#rpcTimeoutMs);
       const leaderId = this.#knownLeader;
       this.#pending.set(requestId, {
@@ -738,7 +1127,9 @@ class OpfsBlockStore {
         sentTo: leaderId,
         resolve,
         reject,
-        timer
+        timer,
+        patienceRounds: 0,
+        awaitingPing: false
       });
       this.#pendingRpcBytes += retainedBytes;
       if (leaderId !== void 0)
@@ -777,8 +1168,7 @@ class OpfsBlockStore {
     if (this.#closed)
       return;
     this.#closed = true;
-    if (this.#reacquireTimer !== void 0)
-      clearTimeout(this.#reacquireTimer);
+    this.#clearCoordinationTimers();
     for (const pending of this.#pending.values()) {
       clearTimeout(pending.timer);
       pending.reject(new Error("This OPFS store connection is closed"));
@@ -797,7 +1187,8 @@ class OpfsBlockStore {
     const leader = this.#leader;
     this.#leader = void 0;
     if (leader !== void 0) {
-      void leader.shutdown().catch(() => {
+      void leader.shutdown().catch((error) => {
+        this.#diagnostic(error, "opfs close shutdown");
         leader.crash();
       }).then(() => {
         this.#post({ kind: "released", leaderId: this.#instanceId });
@@ -806,9 +1197,26 @@ class OpfsBlockStore {
       });
       return;
     }
+    if (this.#yielding !== void 0) {
+      void this.#yielding.then(() => {
+        this.#post({ kind: "released", leaderId: this.#instanceId });
+        this.#closeChannels();
+        this.#releaseWhenHandlesClose();
+      });
+      return;
+    }
     this.#closeChannels();
     this.#releaseWhenHandlesClose();
   }
+  #clearCoordinationTimers() {
+    if (this.#reacquireTimer !== void 0)
+      clearTimeout(this.#reacquireTimer);
+    this.#reacquireTimer = void 0;
+    this.#stopKeepalive();
+    this.#clearDeferredBid();
+    this.#clearHiddenIdleTimer();
+    this.#served.clear();
+  }
   #releaseWhenHandlesClose() {
     const release = this.#releaseConnectionLock;
     this.#releaseConnectionLock = void 0;
@@ -846,6 +1254,9 @@ class OpfsBlockStore {
       release();
     };
   }
+  _oldestPendingRequestIdForTests() {
+    return this.#pending.keys().next().value;
+  }
   _resendOldestPendingForTests() {
     const pending = this.#pending.values().next().value;
     if (pending?.sentTo !== void 0)
@@ -863,8 +1274,7 @@ class OpfsBlockStore {
   }
   _crashForTests() {
     this.#closed = true;
-    if (this.#reacquireTimer !== void 0)
-      clearTimeout(this.#reacquireTimer);
+    this.#clearCoordinationTimers();
     for (const pending of this.#pending.values())
       clearTimeout(pending.timer);
     this.#pending.clear();
@@ -880,7 +1290,11 @@ class OpfsBlockStore {
     this.#servedRequestBytes = 0;
     this.#leader?.crash();
     this.#leader = void 0;
-    this.#closeChannels();
+    this.#channel?.close();
+    this.#channel = void 0;
+    this.#inbox?.close();
+    this.#inbox = void 0;
+    this.#closeAnswerChannels();
     this.#releaseWhenHandlesClose();
   }
   async #ensureFormatMarker() {
@@ -954,23 +1368,35 @@ for (const method of RPC_METHODS) {
     }
   });
 }
+const DELETE_LOCK_WAIT_MS = 1e3;
 async function deleteOpfsDatabase(options) {
   const encodedName = encodeSegment(validateStorageDatabaseName(options.name));
   const locks = globalThis.navigator?.locks;
   if (locks === void 0)
     throw new Error("Deleting an OPFS database requires Web Locks");
-  await locks.request(connectionLockName(options.name), { mode: "exclusive", ifAvailable: true }, async (lock) => {
-    if (lock === null)
+  const abort = new AbortController();
+  const timer = setTimeout(() => {
+    abort.abort();
+  }, DELETE_LOCK_WAIT_MS);
+  try {
+    await locks.request(connectionLockName(options.name), { mode: "exclusive", signal: abort.signal }, async () => {
+      const root = options.root ?? await navigator.storage.getDirectory();
+      try {
+        const namespace = await root.getDirectoryHandle("minnowdb");
+        await namespace.removeEntry(encodedName, { recursive: true });
+      } catch (error) {
+        if (!isDomError(error, "NotFoundError"))
+          throw error;
+      }
+    });
+  } catch (error) {
+    if (abort.signal.aborted && isDomError(error, "AbortError")) {
       throw new OpfsDatabaseInUseError(options.name);
-    const root = options.root ?? await navigator.storage.getDirectory();
-    try {
-      const namespace = await root.getDirectoryHandle("minnowdb");
-      await namespace.removeEntry(encodedName, { recursive: true });
-    } catch (error) {
-      if (!isDomError(error, "NotFoundError"))
-        throw error;
     }
-  });
+    throw error;
+  } finally {
+    clearTimeout(timer);
+  }
 }
 function connectionLockName(name) {
   return `minnowdb-opfs-connections:${name}`;
@@ -985,7 +1411,12 @@ async function holdConnectionLock(name) {
     })).catch(reject);
   });
 }
+function servedRequestKey(from, requestId) {
+  return `${String(from.length)}:${from}${requestId}`;
+}
 const RPC_TIMED_OUT = new Error("The leader did not answer in time");
+const RPC_DECLINED = new Error("The connection asked is not the leader");
+const DECLINED_OUTCOME = { ok: false };
 async function resolveDatabaseRoot(options) {
   const encodedName = encodeSegment(validateStorageDatabaseName(options.name));
   const root = options.root ?? await navigator.storage.getDirectory();