ethers-rpc-pool 1.1.3 → 3.2.0

package/dist/index.js

@@ -9,6 +9,7 @@ var Stats = class {
   _perProviderMethod = {};
   _rateLimitedTotal = 0;
   _timeoutTotal = 0;
+  _serverErrorTotal = 0;
   _rpcErrorTotal = 0;
   _perProviderInFlight = {};
   _perProviderTotal = {};
@@ -17,79 +18,55 @@ var Stats = class {
   _perProviderError = {};
   _perProviderRpcError = {};
   _perMethodRpcError = {};
-  _providerCooldownUntil = {};
   _bump(map, key) {
     map[key] = (map[key] || 0) + 1;
   }
   _decrease(map, key) {
     map[key] = Math.max((map[key] || 0) - 1, 0);
   }
-  _bumpTotal() {
-    this._total++;
-  }
-  _bumpInFlight() {
-    this._inFlight++;
-  }
-  _bumpRateLimitedTotal() {
-    this._rateLimitedTotal++;
-  }
-  _bumpTimeoutTotal() {
-    this._timeoutTotal++;
-  }
-  _bumpRpcErrorTotal() {
-    this._rpcErrorTotal++;
-  }
-  bumpInFlightPerProvider(id) {
-    this._bumpInFlight();
-    this._bump(this._perProviderInFlight, id);
-  }
-  decreaseInFlightPerProvider(id) {
-    this.decreaseInFlight();
-    this._decrease(this._perProviderInFlight, id);
-  }
-  decreaseInFlight() {
+  onEvent(e) {
+    const id = e.providerId;
+    if (e.type === "request") {
+      this._inFlight++;
+      this._bump(this._perProviderInFlight, id);
+      this._total++;
+      this._perProviderTotal[id] = (this._perProviderTotal[id] || 0) + 1;
+      this._bump(this._perMethod, e.method);
+      if (!this._perProviderMethod[id]) this._perProviderMethod[id] = {};
+      this._bump(this._perProviderMethod[id], e.method);
+      return;
+    }
     this._inFlight = Math.max(this._inFlight - 1, 0);
-  }
-  bumpPerMethod(id, method) {
-    this._bump(this._perMethod, method);
-    if (!this._perProviderMethod[id]) {
-      this._perProviderMethod[id] = {};
+    this._decrease(this._perProviderInFlight, id);
+    if (e.type === "error") {
+      if (e.isRateLimit) {
+        this._rateLimitedTotal++;
+        this._bump(this._perProviderRateLimited, id);
+      } else if (e.isTimeout) {
+        this._timeoutTotal++;
+        this._bump(this._perProviderTimeout, id);
+      } else if (e.status !== void 0 && e.status >= 500) {
+        this._serverErrorTotal++;
+        this._bump(this._perProviderError, id);
+      }
     }
-    this._bump(this._perProviderMethod[id], method);
-  }
-  bumpRateLimitedPerProvider(id) {
-    this._bumpRateLimitedTotal();
-    this._bump(this._perProviderRateLimited, id);
-  }
-  bumpTimeoutPerProvider(id) {
-    this._bumpTimeoutTotal();
-    this._bump(this._perProviderTimeout, id);
-  }
-  bumpProviderTotal(id) {
-    this._bumpTotal();
-    this._perProviderTotal[id] = (this._perProviderTotal[id] || 0) + 1;
-  }
-  bumpServerErrorPerProvider(id) {
-    this._bump(this._perProviderError, id);
   }
   bumpRpcError(providerId, method) {
-    this._bumpRpcErrorTotal();
+    this._rpcErrorTotal++;
     if (!this._perProviderRpcError[providerId]) {
       this._perProviderRpcError[providerId] = {};
     }
     this._bump(this._perProviderRpcError[providerId], method);
     this._bump(this._perMethodRpcError, method);
   }
-  timeoutRatio(id) {
-    const t = this._perProviderTimeout[id] || 0;
-    const n = this._perProviderTotal[id] || 0;
-    return n ? t / n : 0;
-  }
-  isInCooldown(id) {
-    return (this._providerCooldownUntil[id] || 0) > Date.now();
-  }
-  setCooldown(id, ms) {
-    this._providerCooldownUntil[id] = Date.now() + ms;
+  removeProvider(id) {
+    delete this._perProviderInFlight[id];
+    delete this._perProviderTotal[id];
+    delete this._perProviderTimeout[id];
+    delete this._perProviderRateLimited[id];
+    delete this._perProviderError[id];
+    delete this._perProviderRpcError[id];
+    delete this._perProviderMethod[id];
   }
   snapshot() {
     return {
@@ -98,16 +75,20 @@ var Stats = class {
       perMethodTotal: { ...this._perMethod },
       rateLimitedTotal: this._rateLimitedTotal,
       timeoutTotal: this._timeoutTotal,
+      serverErrorTotal: this._serverErrorTotal,
       rpcErrorTotal: this._rpcErrorTotal,
-      perProviderMethod: { ...this._perProviderMethod },
+      perProviderMethod: Object.fromEntries(
+        Object.entries(this._perProviderMethod).map(([k, v]) => [k, { ...v }])
+      ),
       perProviderInFlight: { ...this._perProviderInFlight },
       perProviderRateLimited: { ...this._perProviderRateLimited },
       perProviderTimeout: { ...this._perProviderTimeout },
       perProviderError: { ...this._perProviderError },
-      perProviderRpcError: { ...this._perProviderRpcError },
+      perProviderRpcError: Object.fromEntries(
+        Object.entries(this._perProviderRpcError).map(([k, v]) => [k, { ...v }])
+      ),
       perMethodRpcError: { ...this._perMethodRpcError },
-      perProviderTotal: { ...this._perProviderTotal },
-      providerCooldownUntil: { ...this._providerCooldownUntil }
+      perProviderTotal: { ...this._perProviderTotal }
     };
   }
 };
@@ -139,8 +120,12 @@ function isTimeoutError(e) {
   const msg = String(e?.message || e);
   return /timeout|timed out|ETIMEDOUT|ESOCKETTIMEDOUT|ECONNABORTED|504 Gateway/i.test(msg);
 }
+var NETWORK_ERROR_CODES = /* @__PURE__ */ new Set(["ECONNREFUSED", "ECONNRESET", "ENOTFOUND", "EAI_AGAIN"]);
+function isNetworkError(e) {
+  return NETWORK_ERROR_CODES.has(e?.code);
+}
 function isTransportError(e) {
-  return isTimeoutError(e) || isRateLimitError(e) || isServerError(e);
+  return isTimeoutError(e) || isRateLimitError(e) || isServerError(e) || isNetworkError(e);
 }
 function isRpcLogicalError(e) {
   if (isTransportError(e)) return false;
@@ -153,10 +138,7 @@ function isRpcLogicalError(e) {
   );
 }
 function shouldFailover(e) {
-  const to = isTimeoutError(e);
-  const rl = isRateLimitError(e);
-  const se = isServerError(e);
-  return to || rl || se;
+  return isTimeoutError(e) || isRateLimitError(e) || isServerError(e) || isNetworkError(e);
 }
 
 // src/InstrumentedProvider.ts
@@ -235,20 +217,16 @@ var RpsLimiter = class {
     return this.tokens >= count;
   }
   // Take count tokens (usually 1 request = 1 token).
-  // If not enough tokens — wait and try again.
+  // If not enough tokens — enqueue exactly one setTimeout for the precise wake-up time.
+  // tokens may go negative (debt); each caller's wait time is derived from that debt.
   async take(count = 1) {
     if (!this.rps || this.rps <= 0) return;
-    while (true) {
-      const now = Date.now();
-      this.refill(now);
-      if (this.tokens >= count) {
-        this.tokens -= count;
-        return;
-      }
-      const need = count - this.tokens;
-      const waitMs = Math.ceil(need / this.rps * 1e3);
-      await new Promise((r) => setTimeout(r, Math.min(waitMs, 50)));
-    }
+    const now = Date.now();
+    this.refill(now);
+    this.tokens -= count;
+    if (this.tokens >= 0) return;
+    const waitMs = Math.ceil(-this.tokens / this.rps * 1e3);
+    return new Promise((resolve) => setTimeout(resolve, waitMs));
   }
 };
 
@@ -259,9 +237,7 @@ var InstrumentedJsonRpcProvider = class extends JsonRpcProvider {
   options;
   inFlightLimiter;
   rpsLimiter;
-  stats;
   fetchRequest;
-  lastCooldownMs = 0;
   constructor(url, network, options) {
     let fetchRequest;
     if (typeof url == "string") {
@@ -279,29 +255,23 @@ var InstrumentedJsonRpcProvider = class extends JsonRpcProvider {
     const { rps = 10, rpsBurst, inFlight = 1 } = options;
     this.inFlightLimiter = new Semaphore(inFlight);
     this.rpsLimiter = new RpsLimiter(rps, rpsBurst || rps);
-    this.stats = options.stats;
   }
   async _send(payload) {
     await this.rpsLimiter.take(1);
-    const release = this.inFlightLimiter ? await this.inFlightLimiter.acquire() : void 0;
+    const release = await this.inFlightLimiter.acquire();
     try {
       return await this._sendInstrumented(payload);
     } finally {
-      release?.();
+      release();
     }
   }
   isAvailable(count = 1) {
     return this.rpsLimiter.isAvailable(count) && this.inFlightLimiter.isAvailable();
   }
-  // ethers v5 calls send(method, params)
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   async _sendInstrumented(payload) {
     const startedAt = Date.now();
     const payloads = Array.isArray(payload) ? payload : [payload];
-    this.stats.bumpInFlightPerProvider(this.providerId);
-    this.stats.bumpProviderTotal(this.providerId);
     for (const p of payloads) {
-      this.stats.bumpPerMethod(this.providerId, p.method);
       this.options.onEvent?.({
         type: "request",
         chainId: this.chainId,
@@ -324,36 +294,13 @@ var InstrumentedJsonRpcProvider = class extends JsonRpcProvider {
           ms: endedAt - startedAt
         });
       }
-      this.lastCooldownMs = 0;
       return res;
     } catch (e) {
       const endedAt = Date.now();
       const rl = isRateLimitError(e);
-      if (rl) {
-        this.stats.bumpRateLimitedPerProvider(this.providerId);
-        const cooldownMs = 1e4;
-        const raMs = getRetryAfterMs(e) ?? cooldownMs;
-        this.stats.setCooldown(this.providerId, raMs);
-      }
       const isTimeout = isTimeoutError(e);
-      if (isTimeout && !rl) {
-        this.stats.bumpTimeoutPerProvider(this.providerId);
-        const n = this.stats.snapshot().perProviderTotal[this.providerId] || 0;
-        const ratio = this.stats.timeoutRatio(this.providerId);
-        if (n >= 50 && ratio >= 0.2) {
-          const cooldownMs = ratio >= 0.5 ? 6e5 : 6e4;
-          const raMs = getRetryAfterMs(e) ?? cooldownMs;
-          this.lastCooldownMs = raMs;
-          this.stats.setCooldown(this.providerId, raMs + Math.floor(Math.random() * 1e3));
-        }
-      }
-      const isError = isServerError(e);
-      if (isError && !rl && !isTimeout) {
-        this.stats.bumpServerErrorPerProvider(this.providerId);
-        const cooldownMs = (this.lastCooldownMs * 2 || 1e4) + Math.floor(Math.random() * 1e3);
-        this.lastCooldownMs = cooldownMs;
-        this.stats.setCooldown(this.providerId, cooldownMs);
-      }
+      const ne = isNetworkError(e);
+      const retryAfterMs = getRetryAfterMs(e) ?? void 0;
       for (const p of payloads) {
         this.options.onEvent?.({
           type: "error",
@@ -365,37 +312,174 @@ var InstrumentedJsonRpcProvider = class extends JsonRpcProvider {
           ms: endedAt - startedAt,
           isRateLimit: rl,
           isTimeout,
+          isNetworkError: ne,
           status: getHttpStatus(e),
+          retryAfterMs,
           code: e?.code,
           message: String(e?.message || e),
           errorKind: "transport"
         });
       }
       throw e;
-    } finally {
-      this.stats.decreaseInFlightPerProvider(this.providerId);
     }
   }
 };
 
 // src/Router.ts
+var EWMA_ALPHA = 0.2;
 var Router = class {
-  constructor(endpoints, stats) {
-    this.endpoints = endpoints;
-    this.stats = stats;
+  constructor(inputs, availability) {
+    this.availability = availability;
+    const byPriority = /* @__PURE__ */ new Map();
+    for (const { endpoint, priority } of inputs) {
+      if (!byPriority.has(priority)) byPriority.set(priority, []);
+      byPriority.get(priority).push({ endpoint });
+    }
+    this._groups = [...byPriority.entries()].sort(([a], [b]) => b - a).map(([, entries]) => ({ entries, rr: 0 }));
+    this._size = inputs.length;
   }
-  rr = 0;
+  _groups;
+  _size;
+  _ewma = /* @__PURE__ */ new Map();
   size() {
-    return this.endpoints.length;
+    return this._size;
+  }
+  totalSlots() {
+    return this._size;
+  }
+  recordLatency(providerId, ms) {
+    const prev = this._ewma.get(providerId);
+    this._ewma.set(providerId, prev === void 0 ? ms : EWMA_ALPHA * ms + (1 - EWMA_ALPHA) * prev);
+  }
+  ewmaSnapshot() {
+    return Object.fromEntries(this._ewma);
   }
   pick() {
-    const n = this.endpoints.length;
-    for (let k = 0; k < n; k++) {
-      const i = (this.rr++ % n + n) % n;
-      const ep = this.endpoints[i];
-      if (!this.stats.isInCooldown(ep.providerId) && ep.provider.isAvailable(1)) return ep;
+    for (const group of this._groups) {
+      const ep = this._pickFromGroup(group);
+      if (ep !== null) return ep;
+    }
+    const g = this._groups[0];
+    const entry = g.entries[g.rr % g.entries.length];
+    g.rr++;
+    return entry.endpoint;
+  }
+  _pickFromGroup(group) {
+    const available = group.entries.filter(
+      (e) => !this.availability.isInCooldown(e.endpoint.providerId) && e.endpoint.provider.isAvailable(1)
+    );
+    if (available.length === 0) return null;
+    if (available.length === 1) return available[0].endpoint;
+    const i = Math.floor(Math.random() * available.length);
+    let j = Math.floor(Math.random() * (available.length - 1));
+    if (j >= i) j++;
+    const a = available[i].endpoint;
+    const b = available[j].endpoint;
+    const ewmaA = this._ewma.get(a.providerId) ?? 0;
+    const ewmaB = this._ewma.get(b.providerId) ?? 0;
+    return ewmaA <= ewmaB ? a : b;
+  }
+};
+
+// src/CooldownManager.ts
+var CooldownManager = class {
+  _cooldownUntil = {};
+  _lastCooldownMs = {};
+  _perProviderTotal = {};
+  _perProviderTimeout = {};
+  _circuitState = {};
+  _probeInFlight = /* @__PURE__ */ new Set();
+  // atomically claims the probe slot when entering half-open
+  isInCooldown(id) {
+    const state = this._circuitState[id];
+    if (!state) return false;
+    if (state === "half-open") {
+      if (!this._probeInFlight.has(id)) {
+        this._probeInFlight.add(id);
+        return false;
+      }
+      return true;
+    }
+    const until = this._cooldownUntil[id];
+    if (until === void 0 || until <= Date.now()) {
+      delete this._cooldownUntil[id];
+      this._circuitState[id] = "half-open";
+      this._probeInFlight.add(id);
+      return false;
+    }
+    return true;
+  }
+  _openCircuit(id, ms) {
+    this._cooldownUntil[id] = Date.now() + ms;
+    this._circuitState[id] = "open";
+    this._probeInFlight.delete(id);
+  }
+  _openWithBackoff(id) {
+    const prev = this._lastCooldownMs[id] || 0;
+    const ms = (prev ? prev * 2 : 1e4) + Math.floor(Math.random() * 1e3);
+    this._lastCooldownMs[id] = ms;
+    this._openCircuit(id, ms);
+  }
+  cooldownSnapshot() {
+    return { ...this._cooldownUntil };
+  }
+  circuitStateSnapshot() {
+    const result = {};
+    for (const [id, state] of Object.entries(this._circuitState)) {
+      result[id] = state;
+    }
+    return result;
+  }
+  removeProvider(id) {
+    delete this._cooldownUntil[id];
+    delete this._lastCooldownMs[id];
+    delete this._perProviderTotal[id];
+    delete this._perProviderTimeout[id];
+    delete this._circuitState[id];
+    this._probeInFlight.delete(id);
+  }
+  onEvent(e) {
+    const id = e.providerId;
+    if (e.type === "request") {
+      this._perProviderTotal[id] = (this._perProviderTotal[id] || 0) + 1;
+      return;
+    }
+    if (e.type === "response") {
+      if (this._circuitState[id] === "half-open") {
+        delete this._circuitState[id];
+        this._probeInFlight.delete(id);
+      }
+      this._lastCooldownMs[id] = 0;
+      return;
+    }
+    if (this._circuitState[id] === "half-open") {
+      if (e.isRateLimit) {
+        this._openCircuit(id, e.retryAfterMs ?? 1e4);
+      } else {
+        this._openWithBackoff(id);
+      }
+      return;
+    }
+    if (e.isRateLimit) {
+      this._openCircuit(id, e.retryAfterMs ?? 1e4);
+      return;
+    }
+    if (e.isTimeout) {
+      this._perProviderTimeout[id] = (this._perProviderTimeout[id] || 0) + 1;
+      const n = this._perProviderTotal[id] || 0;
+      const t = this._perProviderTimeout[id];
+      const ratio = n ? t / n : 0;
+      if (n >= 50 && ratio >= 0.2) {
+        const baseCooldown = ratio >= 0.5 ? 6e5 : 6e4;
+        const ms = (e.retryAfterMs ?? baseCooldown) + Math.floor(Math.random() * 1e3);
+        this._lastCooldownMs[id] = ms;
+        this._openCircuit(id, ms);
+      }
+      return;
+    }
+    if (e.status !== void 0 && e.status >= 500 || e.isNetworkError) {
+      this._openWithBackoff(id);
     }
-    return this.endpoints[(this.rr++ % n + n) % n];
   }
 };
 
@@ -403,35 +487,55 @@ var Router = class {
 var RPCPoolProvider = class extends JsonRpcProvider2 {
   router;
   params;
-  stats;
+  _stats;
+  _cooldown;
+  _endpoints;
+  _probeInterval;
   constructor(params) {
     const network = Network2.from(params.network);
     super("http://localhost", network, { staticNetwork: network });
     this.params = params;
-    this.stats = new Stats();
-    const endpoints = this.params.rpc.map((options, i) => {
+    this._stats = new Stats();
+    this._cooldown = new CooldownManager();
+    const endpoints = [];
+    const routerInputs = this.params.rpc.map((options, i) => {
+      const { priority, network: _n, ...providerOptions } = options;
       const url = typeof options.url === "string" ? options.url : options.url.url;
       const providerId = `rpc#${i + 1}-chainId:${this.params.network}-${url}`;
       const provider = new InstrumentedJsonRpcProvider(options.url, this.params.network, {
         providerId,
-        stats: this.stats,
         ...this.params.defaultRpcOptions,
-        ...options,
-        onEvent: this.params.hooks?.onEvent
+        ...providerOptions,
+        onEvent: (e) => this._handleTransportEvent(e)
       });
-      return { providerId, url, provider };
+      endpoints.push({ id: providerId, provider });
+      const endpoint = { providerId, url, provider };
+      return { endpoint, priority: priority ?? 0 };
     });
-    this.router = new Router(endpoints, this.stats);
+    this._endpoints = endpoints;
+    this.router = new Router(routerInputs, this._cooldown);
+    if (params.healthProbe !== void 0) {
+      const intervalMs = params.healthProbe.intervalMs ?? 15e3;
+      this._probeInterval = setInterval(() => this._runHealthProbe(), intervalMs);
+      this._probeInterval.unref?.();
+    }
+  }
+  destroy() {
+    if (this._probeInterval !== void 0) {
+      clearInterval(this._probeInterval);
+      this._probeInterval = void 0;
+    }
   }
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   async send(method, params) {
+    if (this.router.size() === 0) throw new Error("No RPC available");
     const tried = /* @__PURE__ */ new Set();
     const maxAttempts = this.params.retry.attempts;
     for (let attempt = 0; attempt < maxAttempts; attempt++) {
       let endpoint = this.router.pick();
       if (tried.size < this.router.size()) {
         let retries = 0;
-        while (tried.has(endpoint.providerId) && retries < this.router.size()) {
+        while (tried.has(endpoint.providerId) && retries < this.router.totalSlots()) {
           endpoint = this.router.pick();
           retries++;
         }
@@ -442,23 +546,60 @@ var RPCPoolProvider = class extends JsonRpcProvider2 {
         return await endpoint.provider.send(method, params);
       } catch (e) {
         if (isRpcLogicalError(e)) {
-          this.emitRpcLogicalError(endpoint, method, startedAt, e);
+          this._emitRpcLogicalError(endpoint, method, startedAt, e);
         }
         if (!shouldFailover(e)) throw e;
         if (attempt === maxAttempts - 1) throw e;
-        await this.sleepWithBackoff(attempt);
+        await this._sleepWithBackoff(attempt);
       }
     }
     throw new Error("No RPC available");
   }
-  async sleepWithBackoff(attempt) {
+  getSnapshot() {
+    return {
+      ...this._stats.snapshot(),
+      providerCooldownUntil: this._cooldown.cooldownSnapshot(),
+      providerCircuitState: this._cooldown.circuitStateSnapshot(),
+      perProviderLatencyEwma: this.router.ewmaSnapshot()
+    };
+  }
+  // Returns a provider pinned to a single RPC node selected right now.
+  // Use when consecutive calls must see consistent chain state — e.g. read
+  // eth_getBalance on block N, then eth_call on the same block N. Without
+  // pinning, each call may route to a different node that lags behind.
+  pinnedProvider() {
+    return this.router.pick().provider;
+  }
+  _runHealthProbe() {
+    const states = this._cooldown.circuitStateSnapshot();
+    const cooldowns = this._cooldown.cooldownSnapshot();
+    const now = Date.now();
+    for (const { id, provider } of this._endpoints) {
+      if (states[id] !== "open") continue;
+      const cooldownUntil = cooldowns[id];
+      if (cooldownUntil !== void 0 && cooldownUntil > now) continue;
+      if (!this._cooldown.isInCooldown(id)) {
+        provider.send("eth_blockNumber", []).catch(() => {
+        });
+      }
+    }
+  }
+  _handleTransportEvent(e) {
+    this._stats.onEvent(e);
+    this._cooldown.onEvent(e);
+    if (e.type === "response" || e.type === "error") {
+      this.router.recordLatency(e.providerId, e.ms);
+    }
+    this.params.hooks?.onEvent?.(e);
+  }
+  async _sleepWithBackoff(attempt) {
     const baseDelay = Math.min(1e3 * 2 ** attempt, 5e3);
     const jitter = Math.random() * baseDelay;
     await new Promise((resolve) => setTimeout(resolve, jitter));
   }
-  emitRpcLogicalError(ep, method, startedAt, error) {
+  _emitRpcLogicalError(ep, method, startedAt, error) {
     const endedAt = Date.now();
-    this.stats.bumpRpcError(ep.providerId, method);
+    this._stats.bumpRpcError(ep.providerId, method);
     this.params.hooks?.onEvent?.({
       type: "error",
       chainId: BigInt(Network2.from(this.params.network).chainId),
@@ -469,15 +610,13 @@ var RPCPoolProvider = class extends JsonRpcProvider2 {
       ms: endedAt - startedAt,
       isRateLimit: false,
       isTimeout: false,
+      isNetworkError: false,
       status: void 0,
       code: error?.code,
       message: String(error?.message || error),
       errorKind: "rpc"
     });
   }
-  getStats() {
-    return this.stats;
-  }
 };
 export {
   RPCPoolProvider