@pioneer-platform/eth-network 8.41.19 → 8.41.20

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
 
-> @pioneer-platform/eth-network@8.41.19 build /Users/highlander/WebstormProjects/keepkey-stack/projects/pioneer/modules/coins/eth/eth-network
+> @pioneer-platform/eth-network@8.41.20 build /Users/highlander/WebstormProjects/keepkey-stack/projects/pioneer/modules/coins/eth/eth-network
 > tsc -p .
 

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @pioneer-platform/eth-network
 
+## 8.41.20
+
+### Patch Changes
+
+- d1d2546: Production hotfix sweep landed via #28, #29, #30, #31, #32:
+
+  - `eth-network`: detect structurally-dead EVM RPCs via error-body signatures ("no longer available", "has been sunset", etc.) and quarantine for 24h instead of retrying every 5 min. Stop misclassifying transient network failures (ECONNRESET, timeout, 5xx wrapped as CALL_EXCEPTION) as contract-not-found — was reporting USDT as nonexistent on Ethereum.
+  - `evm-network`: pin `staticNetwork` on `JsonRpcProvider` and `destroy()` after use, so a failed detect doesn't spin up ethers v6's unbounded retry loop and leak a log-spamming async task per request.
+  - `pioneer-nodes`: filter known-dead providers (`*.public.blastapi.io`, `1rpc.io`) at `getWeb3Nodes()` boundary via regex.
+
+- Updated dependencies [d1d2546]
+  - @pioneer-platform/pioneer-nodes@8.37.11
+  - @pioneer-platform/blockbook@8.38.20
+
 ## 8.41.19
 
 ### Patch Changes

package/lib/index.js

@@ -104,6 +104,50 @@ let NODES = [];
 // Dead node tracker - nodes that failed recently (service URL -> { timestamp, ttl })
 const DEAD_NODES = new Map();
 const DEAD_NODE_TTL_MS = parseInt(process.env.ETH_DEAD_NODE_TTL || '300000'); // 5 minutes before retry
+// Long quarantine TTL for nodes that return a structural-failure signature
+// (provider says "discontinued" / "no longer available" / "sunset"). 24h so
+// the dead TTL doesn't resurrect them every 5 minutes.
+const STRUCTURAL_FAILURE_TTL_MS = 24 * 60 * 60 * 1000;
+// Nodes we've already logged as "permanently" failing, so we don't spam
+// the same WARN every time a caller tries to mark the same URL dead.
+// Cleared on pod restart.
+const LOGGED_STRUCTURAL_FAILURE = new Set();
+/**
+ * Detect error responses that indicate the provider is structurally gone —
+ * the endpoint explicitly says it's discontinued, not that it's merely having
+ * a bad moment. Matched strings come straight from the JSON-RPC error body
+ * or axios message. When this returns true we quarantine for 24h on the
+ * first failure instead of burning a fresh request every 5 minutes.
+ *
+ * Examples we want to catch (case-insensitive):
+ *   "Blast API is no longer available. Please update your integration..."
+ *   "This service has been discontinued"
+ *   "This endpoint has been sunset"
+ */
+const isPermanentProviderFailure = (error) => {
+    const fragments = [];
+    const push = (v) => {
+        if (typeof v === 'string')
+            fragments.push(v);
+    };
+    push(error?.message);
+    push(error?.body);
+    push(error?.response?.body);
+    push(error?.response?.data);
+    push(error?.serverError?.message);
+    push(error?.error?.message);
+    push(error?.error?.body);
+    const haystack = fragments.join(' ').toLowerCase();
+    if (!haystack)
+        return false;
+    return (haystack.includes('no longer available') ||
+        haystack.includes('has been discontinued') ||
+        haystack.includes('has been sunset') ||
+        haystack.includes('has been retired') ||
+        haystack.includes('please update your integration') ||
+        haystack.includes('endpoint is deprecated') ||
+        haystack.includes('service is deprecated'));
+};
 const RACE_BATCH_SIZE = parseInt(process.env.ETH_RACE_BATCH_SIZE || '1'); // Try 1 node at a time (SEQUENTIAL, not parallel racing)
 // NOTE: Changed from 5 to 1 to reduce API request load by 80%
 // Fast timeout (500ms) ensures quick fallback if node is slow/down
@@ -144,23 +188,50 @@ const isNodeDead = (nodeUrl) => {
     return true;
 };
 /**
- * Mark a node as dead with optional custom TTL
- * Only logs at WARN level for premium/reliable nodes, DEBUG for others
- * @param nodeUrl Node service URL
- * @param tier Node tier (for logging)
- * @param customTTL Optional custom TTL in ms (defaults to DEAD_NODE_TTL_MS)
+ * Mark a node as dead with a TTL chosen from these signals, in priority order:
+ *   1. Explicit customTTL (used by startup validation).
+ *   2. Structural-failure signature in the error body ("no longer available",
+ *      "has been sunset", etc.) → 24h quarantine and one-shot log per URL.
+ *   3. Default 5 min TTL.
+ *
+ * We intentionally do NOT adjust the TTL based on consecutive-failure counts
+ * from node-health metrics here — the token-balance and token-metadata hot
+ * paths don't currently call recordSuccess on their success branches, so the
+ * count only ever goes up for those, and intermittent token failures would
+ * ratchet otherwise-healthy nodes into longer and longer quarantines. If
+ * we want automatic escalation later, it needs symmetric success tracking.
  */
-const markNodeDead = (nodeUrl, tier, customTTL) => {
-    // Use custom TTL if provided, otherwise use default
-    const ttl = customTTL || DEAD_NODE_TTL_MS;
+const markNodeDead = (nodeUrl, tier, customTTL, error) => {
+    let ttl = DEAD_NODE_TTL_MS;
+    let reason = 'default';
+    if (customTTL) {
+        ttl = customTTL;
+        reason = 'explicit';
+    }
+    else if (error && isPermanentProviderFailure(error)) {
+        ttl = STRUCTURAL_FAILURE_TTL_MS;
+        reason = 'structural';
+    }
     DEAD_NODES.set(nodeUrl, { timestamp: Date.now(), ttl });
-    // Only log failures for premium/reliable nodes (reduces spam)
+    const logTag = TAG + ' | markNodeDead | ';
     const ttlMinutes = Math.round(ttl / 1000 / 60);
+    const label = tier ?? 'public';
+    const urlShort = nodeUrl.substring(0, 50);
+    // One-shot log for structural failures so we don't repeat the WARN every
+    // time a different caller re-discovers that a known-dead provider is dead.
+    // Reset on pod restart, so it re-appears after redeploy.
+    if (reason === 'structural') {
+        if (!LOGGED_STRUCTURAL_FAILURE.has(nodeUrl)) {
+            LOGGED_STRUCTURAL_FAILURE.add(nodeUrl);
+            log.warn(logTag, `🛑 ${label} node quarantined (structural failure): ${urlShort}... dead for ${ttlMinutes}min`);
+        }
+        return;
+    }
     if (tier === 'premium' || tier === 'reliable') {
-        log.warn(TAG + ' | markNodeDead | ', `⚠️  ${tier} node failed: ${nodeUrl.substring(0, 50)}... (dead for ${ttlMinutes}min)`);
+        log.warn(logTag, `⚠️  ${label} node failed: ${urlShort}... (dead for ${ttlMinutes}min)`);
     }
     else {
-        log.debug(TAG + ' | markNodeDead | ', `Public/untrusted node failed: ${nodeUrl.substring(0, 50)}... (dead for ${ttlMinutes}min)`);
+        log.debug(logTag, `Public/untrusted node failed: ${urlShort}... (dead for ${ttlMinutes}min)`);
     }
 };
 /**
@@ -879,7 +950,7 @@ const getBalanceAddressByNetwork = async function (networkId, address) {
                     e.code === 'CERT_HAS_EXPIRED' ||
                     e.code === 'SELF_SIGNED_CERT_IN_CHAIN');
                 if (!isValidationError && !isSSLError) {
-                    markNodeDead(node.service, node.tier);
+                    markNodeDead(node.service, node.tier, undefined, e);
                     await NodeHealth.recordFailure(node.service, node.networkId);
                 }
                 else if (isValidationError) {
@@ -957,6 +1028,14 @@ const getBalanceTokenByNetwork = async function (networkId, address, tokenAddres
             catch (e) {
                 // Check error type before marking node dead
                 log.warn(tag, `❌ Node ${batchStartIndex + index + 1}/${nodes.length} failed: ${node.service.substring(0, 50)} - ${e.message}`);
+                // FAIL FAST: Contract doesn't exist on this chain — no point trying other nodes
+                if (isContractNotFoundError(e)) {
+                    log.error(tag, `Contract not found on ${networkId} (CALL_EXCEPTION with empty data)`);
+                    const err = new Error(`Contract not found on network ${networkId}. The token may not be deployed on this chain.`);
+                    err.code = 'CONTRACT_NOT_FOUND';
+                    err.statusCode = 404;
+                    throw err;
+                }
                 // Don't mark nodes dead for validation or SSL errors
                 const isValidationError = e.message && (e.message.includes('invalid address') ||
                     e.message.includes('INVALID_ARGUMENT') ||
@@ -969,7 +1048,7 @@ const getBalanceTokenByNetwork = async function (networkId, address, tokenAddres
                     e.code === 'CERT_HAS_EXPIRED' ||
                     e.code === 'SELF_SIGNED_CERT_IN_CHAIN');
                 if (!isValidationError && !isSSLError) {
-                    markNodeDead(node.service, node.tier);
+                    markNodeDead(node.service, node.tier, undefined, e);
                 }
                 else if (isValidationError) {
                     log.warn(tag, `Skipping node death marking - validation error`);
@@ -982,6 +1061,11 @@ const getBalanceTokenByNetwork = async function (networkId, address, tokenAddres
         });
         // Use allSettled to let all nodes complete, return first success
         const results = await Promise.allSettled(racePromises);
+        // FAIL FAST: If any node got CONTRACT_NOT_FOUND, re-throw immediately
+        const contractNotFound = results.find(r => r.status === 'rejected' && r.reason?.code === 'CONTRACT_NOT_FOUND');
+        if (contractNotFound && contractNotFound.status === 'rejected') {
+            throw contractNotFound.reason;
+        }
         // Find first successful result
         const successResult = results.find(r => r.status === 'fulfilled' && r.value.success);
         if (successResult && successResult.status === 'fulfilled') {
@@ -1013,6 +1097,39 @@ const getBalanceTokensByNetwork = async function (networkId, address) {
     }
 };
 exports.getBalanceTokensByNetwork = getBalanceTokensByNetwork;
+/**
+ * Check if an error indicates the contract doesn't exist on this chain.
+ *
+ * ethers v5 wraps any failure during an `eth_call` as `CALL_EXCEPTION` — including
+ * transient network errors like ECONNRESET, timeouts, and 5xx responses, which
+ * arrive with `data="0x"` too. A naive `code === 'CALL_EXCEPTION' && data === '0x'`
+ * check therefore misclassifies network errors as "contract not found" and, when
+ * two such failures happen in a row, causes e.g. USDT to be reported as non-existent
+ * on Ethereum mainnet. See: https://links.ethers.org/v5-errors-CALL_EXCEPTION
+ *
+ * We only conclude "not found" when ethers actually saw a response from the node
+ * (no wrapped SERVER_ERROR / TIMEOUT / NETWORK_ERROR, and no "missing response"/
+ * "bad response" reason).
+ */
+const isContractNotFoundError = (error) => {
+    if (error?.code !== 'CALL_EXCEPTION')
+        return false;
+    // Wrapped transport failures — the RPC never returned an eth_call result.
+    const innerCode = error?.error?.code;
+    if (innerCode === 'SERVER_ERROR' || innerCode === 'TIMEOUT' || innerCode === 'NETWORK_ERROR') {
+        return false;
+    }
+    const reason = typeof error?.reason === 'string' ? error.reason : '';
+    if (reason === 'missing response' || reason === 'bad response' || reason === 'could not detect network') {
+        return false;
+    }
+    // Our own withTimeout wrapper bubbles up as a plain Error with a timeout message.
+    if (typeof error?.message === 'string' && error.message.includes('timeout after')) {
+        return false;
+    }
+    // Genuine contract-not-found: eth_call returned "0x" with no revert data.
+    return !error?.data || error?.data === '0x';
+};
 /**
  * Get ERC20 token decimals by network - CRITICAL for correct amount calculations
  */
@@ -1029,6 +1146,12 @@ const getTokenDecimalsByNetwork = async function (networkId, tokenAddress) {
         chainId = parseInt(networkId.split(':')[1]);
     }
     // Try each node until one succeeds
+    // Require multiple nodes to agree before concluding contract doesn't exist,
+    // because public RPCs can return CALL_EXCEPTION with empty data for rate limiting.
+    // 3 is chosen to tolerate two correlated transient failures (e.g. 1rpc.io
+    // and ankr hiccuping at the same time) without mislabelling real tokens.
+    const MIN_CONTRACT_NOT_FOUND_CONFIRMATIONS = 3;
+    let contractNotFoundCount = 0;
     let lastError = null;
     for (let i = 0; i < nodes.length; i++) {
         const node = nodes[i];
@@ -1046,11 +1169,31 @@ const getTokenDecimalsByNetwork = async function (networkId, tokenAddress) {
         }
         catch (e) {
             lastError = e;
+            if (isContractNotFoundError(e)) {
+                contractNotFoundCount++;
+                log.warn(tag, `Node ${i + 1}/${nodes.length} returned CALL_EXCEPTION for ${tokenAddress} (${contractNotFoundCount}/${MIN_CONTRACT_NOT_FOUND_CONFIRMATIONS} confirmations needed)`);
+                // Only throw CONTRACT_NOT_FOUND if multiple nodes agree
+                if (contractNotFoundCount >= MIN_CONTRACT_NOT_FOUND_CONFIRMATIONS) {
+                    log.error(tag, `Contract ${tokenAddress} confirmed not found on ${networkId} by ${contractNotFoundCount} nodes.`);
+                    const err = new Error(`Contract ${tokenAddress} not found on network ${networkId}. The token may not be deployed on this chain.`);
+                    err.code = 'CONTRACT_NOT_FOUND';
+                    err.statusCode = 404;
+                    throw err;
+                }
+                // Otherwise fall through to try next node
+            }
             log.warn(tag, `Node ${i + 1}/${nodes.length} failed (${node.service.substring(0, 50)}): ${e.message}`);
             // Continue to next node
         }
     }
-    // All nodes failed
+    // All nodes failed — if every failure was CALL_EXCEPTION, it's likely genuinely not found
+    if (contractNotFoundCount > 0 && contractNotFoundCount === nodes.length) {
+        log.error(tag, `All ${nodes.length} nodes returned CALL_EXCEPTION for ${tokenAddress} on ${networkId}`);
+        const err = new Error(`Contract ${tokenAddress} not found on network ${networkId}. The token may not be deployed on this chain.`);
+        err.code = 'CONTRACT_NOT_FOUND';
+        err.statusCode = 404;
+        throw err;
+    }
     log.error(tag, `All ${nodes.length} nodes failed for ${networkId}`);
     throw lastError || Error(`Failed to get token decimals from any node for ${networkId}`);
 };
@@ -1131,6 +1274,14 @@ const getTokenMetadata = async function (networkId, contractAddress, userAddress
             catch (e) {
                 // Check error type before marking node dead
                 log.warn(tag, `❌ Node ${batchStartIndex + index + 1}/${nodes.length} failed: ${node.service.substring(0, 50)} - ${e.message}`);
+                // FAIL FAST: Contract doesn't exist on this chain — no point trying other nodes
+                if (isContractNotFoundError(e)) {
+                    log.error(tag, `Contract not found on ${networkId} (CALL_EXCEPTION with empty data)`);
+                    const err = new Error(`Contract not found on network ${networkId}. The token may not be deployed on this chain.`);
+                    err.code = 'CONTRACT_NOT_FOUND';
+                    err.statusCode = 404;
+                    throw err;
+                }
                 // Don't mark nodes dead for validation or SSL errors
                 const isValidationError = e.message && (e.message.includes('invalid address') ||
                     e.message.includes('INVALID_ARGUMENT') ||
@@ -1143,7 +1294,7 @@ const getTokenMetadata = async function (networkId, contractAddress, userAddress
                     e.code === 'CERT_HAS_EXPIRED' ||
                     e.code === 'SELF_SIGNED_CERT_IN_CHAIN');
                 if (!isValidationError && !isSSLError) {
-                    markNodeDead(node.service, node.tier);
+                    markNodeDead(node.service, node.tier, undefined, e);
                 }
                 else if (isValidationError) {
                     log.warn(tag, `Skipping node death marking - validation error`);
@@ -1156,6 +1307,11 @@ const getTokenMetadata = async function (networkId, contractAddress, userAddress
         });
         // Use allSettled to let all nodes complete, return first success
         const results = await Promise.allSettled(racePromises);
+        // FAIL FAST: If any node got CONTRACT_NOT_FOUND, re-throw immediately
+        const contractNotFound = results.find(r => r.status === 'rejected' && r.reason?.code === 'CONTRACT_NOT_FOUND');
+        if (contractNotFound && contractNotFound.status === 'rejected') {
+            throw contractNotFound.reason;
+        }
         // Find first successful result
         const successResult = results.find(r => r.status === 'fulfilled' && r.value.success);
         if (successResult && successResult.status === 'fulfilled') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pioneer-platform/eth-network",
-  "version": "8.41.19",
+  "version": "8.41.20",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
   "dependencies": {
@@ -17,10 +17,10 @@
     "ethers": "5.7.2",
     "request-promise": "^4.2.6",
     "wait-promise": "^0.4.1",
-    "@pioneer-platform/blockbook": "8.38.19",
-    "@pioneer-platform/pioneer-nodes": "8.37.10",
-    "@pioneer-platform/pioneer-caip": "9.27.10",
-    "@pioneer-platform/loggerdog": "8.11.0"
+    "@pioneer-platform/pioneer-nodes": "8.37.11",
+    "@pioneer-platform/loggerdog": "8.11.0",
+    "@pioneer-platform/blockbook": "8.38.20",
+    "@pioneer-platform/pioneer-caip": "9.27.10"
   },
   "devDependencies": {
     "@types/node": "^18.16.0",