@hiveio/dhive 1.3.2 → 1.3.4

package/lib/client.d.ts

@@ -40,6 +40,7 @@ import { HivemindAPI } from './helpers/hivemind';
 import { AccountByKeyAPI } from './helpers/key';
 import { RCAPI } from './helpers/rc';
 import { TransactionStatusAPI } from './helpers/transaction';
+import { NodeHealthTracker, HealthTrackerOptions } from './health-tracker';
 /**
  * Library version.
  */
@@ -102,6 +103,11 @@ export interface ClientOptions {
      * Deprecated - don't use
      */
     rebrandedApi?: boolean;
+    /**
+     * Options for the node health tracker.
+     * Controls cooldown periods, stale block thresholds, etc.
+     */
+    healthTrackerOptions?: HealthTrackerOptions;
 }
 /**
  * RPC Client
@@ -146,6 +152,11 @@ export declare class Client {
      * Transaction status API helper.
      */
     readonly transaction: TransactionStatusAPI;
+    /**
+     * Node health tracker for smart failover.
+     * Tracks per-node, per-API health and head block freshness.
+     */
+    readonly healthTracker: NodeHealthTracker;
     /**
      * Chain ID for current network.
      */

package/lib/client.js

@@ -53,6 +53,7 @@ const hivemind_1 = require("./helpers/hivemind");
 const key_1 = require("./helpers/key");
 const rc_1 = require("./helpers/rc");
 const transaction_1 = require("./helpers/transaction");
+const health_tracker_1 = require("./health-tracker");
 const utils_1 = require("./utils");
 /**
  * Library version.
@@ -94,6 +95,7 @@ class Client {
         this.backoff = options.backoff || defaultBackoff;
         this.failoverThreshold = options.failoverThreshold || 3;
         this.consoleOnFailover = options.consoleOnFailover || false;
+        this.healthTracker = new health_tracker_1.NodeHealthTracker(options.healthTrackerOptions);
         this.database = new database_1.DatabaseAPI(this);
         this.broadcast = new broadcast_1.BroadcastAPI(this);
         this.blockchain = new blockchain_1.Blockchain(this);
@@ -125,6 +127,8 @@ class Client {
      */
     call(api, method, params = []) {
         return __awaiter(this, void 0, void 0, function* () {
+            const isBroadcast = api === 'network_broadcast_api' ||
+                method.startsWith('broadcast_transaction');
             const request = {
                 id: 0,
                 jsonrpc: '2.0',
@@ -159,18 +163,32 @@ class Client {
                 opts.agent = this.options.agent;
             }
             let fetchTimeout;
-            if (api !== 'network_broadcast_api' &&
-                !method.startsWith('broadcast_transaction')) {
+            if (!isBroadcast) {
                 // bit of a hack to work around some nodes high error rates
                 // only effective in node.js (until timeout spec lands in browsers)
                 fetchTimeout = (tries) => (tries + 1) * 500;
             }
-            const { response, currentAddress } = yield utils_1.retryingFetch(this.currentAddress, this.address, opts, this.timeout, this.failoverThreshold, this.consoleOnFailover, this.backoff, fetchTimeout);
+            const { response, currentAddress } = yield utils_1.retryingFetch(this.currentAddress, this.address, opts, this.timeout, this.failoverThreshold, this.consoleOnFailover, this.backoff, fetchTimeout, {
+                healthTracker: this.healthTracker,
+                api,
+                isBroadcast,
+                consoleOnFailover: this.consoleOnFailover,
+            });
             // After failover, change the currently active address
             if (currentAddress !== this.currentAddress) {
                 this.currentAddress = currentAddress;
             }
-            // resolve FC error messages into something more readable
+            // Passively track head block from get_dynamic_global_properties responses.
+            // This costs nothing — we just inspect data we already fetched.
+            if (response.result &&
+                method === 'get_dynamic_global_properties' &&
+                response.result.head_block_number) {
+                this.healthTracker.updateHeadBlock(currentAddress, response.result.head_block_number);
+            }
+            // Handle RPC-level errors.
+            // Unlike network errors, these mean the node responded but returned an error.
+            // We record it as an API-specific failure so the health tracker can
+            // deprioritize this node for this API in future calls.
             if (response.error) {
                 const formatValue = (value) => {
                     switch (typeof value) {
@@ -200,6 +218,17 @@ class Client {
                         message += ' ' + unformattedData.join(' ');
                     }
                 }
+                // Track RPC errors that indicate node/plugin issues (not user errors).
+                // JSON-RPC error codes (response.error.code):
+                //   -32601 = Method not found (plugin not enabled on this node)
+                //   -32603 = Internal error (node issue)
+                //   -32003 = Hive assertion error (user error — bad params, invalid account)
+                // Only API/plugin errors should be tracked, and only as API-specific failures
+                // (not global node failures) since other APIs on this node may work fine.
+                const rpcCode = response.error.code;
+                if (rpcCode === -32601 || rpcCode === -32603) {
+                    this.healthTracker.recordApiFailure(currentAddress, api);
+                }
                 throw new verror_1.VError({ info: data, name: 'RPCError' }, message);
             }
             assert.equal(response.id, request.id, 'got invalid response id');

package/lib/health-tracker.d.ts

@@ -0,0 +1,100 @@
+/**
+ * @file Node health tracking for smart failover.
+ * @license BSD-3-Clause-No-Military-License
+ *
+ * Tracks per-node, per-API health to enable intelligent failover decisions.
+ * Nodes that fail for specific APIs are deprioritized for those APIs while
+ * remaining available for others. Stale nodes (behind on head block) are
+ * also deprioritized.
+ */
+export interface HealthTrackerOptions {
+    /**
+     * How long (ms) to deprioritize a node after consecutive failures.
+     * Default: 30 seconds.
+     */
+    nodeCooldownMs?: number;
+    /**
+     * How long (ms) to deprioritize a node for a specific API after failures.
+     * Default: 60 seconds.
+     */
+    apiCooldownMs?: number;
+    /**
+     * Number of consecutive failures before a node enters cooldown.
+     * Default: 3.
+     */
+    maxFailuresBeforeCooldown?: number;
+    /**
+     * Number of API-specific failures before deprioritizing for that API.
+     * Default: 2.
+     */
+    maxApiFailuresBeforeCooldown?: number;
+    /**
+     * How many blocks behind the best known head block a node can be
+     * before being considered stale. Default: 30.
+     */
+    staleBlockThreshold?: number;
+    /**
+     * How long (ms) head block data remains valid for staleness checks.
+     * Default: 2 minutes.
+     */
+    headBlockTtlMs?: number;
+}
+export declare class NodeHealthTracker {
+    private health;
+    private bestKnownHeadBlock;
+    private bestKnownHeadBlockTime;
+    private readonly nodeCooldownMs;
+    private readonly apiCooldownMs;
+    private readonly maxFailuresBeforeCooldown;
+    private readonly maxApiFailuresBeforeCooldown;
+    private readonly staleBlockThreshold;
+    private readonly headBlockTtlMs;
+    constructor(options?: HealthTrackerOptions);
+    private getOrCreate;
+    /**
+     * Record a successful call to a node for a specific API.
+     * Clears consecutive failure counter and API-specific failures for this API.
+     */
+    recordSuccess(node: string, api: string): void;
+    /**
+     * Record a network-level failure (timeout, connection refused, HTTP error).
+     * Increments both the global consecutive failure counter and the API-specific counter.
+     */
+    recordFailure(node: string, api: string): void;
+    /**
+     * Record an API/plugin-specific failure (e.g. "method not found", "plugin not enabled").
+     * Only increments the per-API counter, NOT the global consecutive failure counter.
+     * This prevents a node with a disabled plugin from being penalized for all APIs.
+     */
+    recordApiFailure(node: string, api: string): void;
+    private incrementApiFailure;
+    /**
+     * Update head block number for a node.
+     * Called passively when get_dynamic_global_properties responses are observed.
+     */
+    updateHeadBlock(node: string, headBlock: number): void;
+    /**
+     * Check if a node is considered healthy for a given API.
+     */
+    isNodeHealthy(node: string, api?: string): boolean;
+    /**
+     * Return nodes ordered by health for a specific API call.
+     * Healthy nodes come first (preserving original order), then unhealthy nodes as fallback.
+     */
+    getOrderedNodes(allNodes: string[], api?: string): string[];
+    /**
+     * Reset all health tracking data.
+     */
+    reset(): void;
+    /**
+     * Get a snapshot of current health state for diagnostics.
+     */
+    getHealthSnapshot(): Map<string, {
+        consecutiveFailures: number;
+        headBlock: number;
+        apiFailures: Record<string, {
+            count: number;
+        }>;
+        healthy: boolean;
+    }>;
+}

package/lib/health-tracker.js

@@ -0,0 +1,167 @@
+"use strict";
+/**
+ * @file Node health tracking for smart failover.
+ * @license BSD-3-Clause-No-Military-License
+ *
+ * Tracks per-node, per-API health to enable intelligent failover decisions.
+ * Nodes that fail for specific APIs are deprioritized for those APIs while
+ * remaining available for others. Stale nodes (behind on head block) are
+ * also deprioritized.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+class NodeHealthTracker {
+    constructor(options = {}) {
+        var _a, _b, _c, _d, _e, _f;
+        this.health = new Map();
+        this.bestKnownHeadBlock = 0;
+        this.bestKnownHeadBlockTime = 0;
+        this.nodeCooldownMs = (_a = options.nodeCooldownMs) !== null && _a !== void 0 ? _a : 30000;
+        this.apiCooldownMs = (_b = options.apiCooldownMs) !== null && _b !== void 0 ? _b : 60000;
+        this.maxFailuresBeforeCooldown = (_c = options.maxFailuresBeforeCooldown) !== null && _c !== void 0 ? _c : 3;
+        this.maxApiFailuresBeforeCooldown = (_d = options.maxApiFailuresBeforeCooldown) !== null && _d !== void 0 ? _d : 2;
+        this.staleBlockThreshold = (_e = options.staleBlockThreshold) !== null && _e !== void 0 ? _e : 30;
+        this.headBlockTtlMs = (_f = options.headBlockTtlMs) !== null && _f !== void 0 ? _f : 120000;
+    }
+    getOrCreate(node) {
+        let state = this.health.get(node);
+        if (!state) {
+            state = {
+                apiFailures: new Map(),
+                consecutiveFailures: 0,
+                lastFailure: 0,
+                headBlock: 0,
+                headBlockUpdatedAt: 0,
+            };
+            this.health.set(node, state);
+        }
+        return state;
+    }
+    /**
+     * Record a successful call to a node for a specific API.
+     * Clears consecutive failure counter and API-specific failures for this API.
+     */
+    recordSuccess(node, api) {
+        const state = this.getOrCreate(node);
+        state.consecutiveFailures = 0;
+        state.apiFailures.delete(api);
+    }
+    /**
+     * Record a network-level failure (timeout, connection refused, HTTP error).
+     * Increments both the global consecutive failure counter and the API-specific counter.
+     */
+    recordFailure(node, api) {
+        const state = this.getOrCreate(node);
+        state.consecutiveFailures++;
+        state.lastFailure = Date.now();
+        this.incrementApiFailure(state, api);
+    }
+    /**
+     * Record an API/plugin-specific failure (e.g. "method not found", "plugin not enabled").
+     * Only increments the per-API counter, NOT the global consecutive failure counter.
+     * This prevents a node with a disabled plugin from being penalized for all APIs.
+     */
+    recordApiFailure(node, api) {
+        const state = this.getOrCreate(node);
+        this.incrementApiFailure(state, api);
+    }
+    incrementApiFailure(state, api) {
+        const apiState = state.apiFailures.get(api) || { count: 0, lastFailure: 0 };
+        apiState.count++;
+        apiState.lastFailure = Date.now();
+        state.apiFailures.set(api, apiState);
+    }
+    /**
+     * Update head block number for a node.
+     * Called passively when get_dynamic_global_properties responses are observed.
+     */
+    updateHeadBlock(node, headBlock) {
+        if (!headBlock || headBlock <= 0)
+            return;
+        const state = this.getOrCreate(node);
+        state.headBlock = headBlock;
+        state.headBlockUpdatedAt = Date.now();
+        if (headBlock > this.bestKnownHeadBlock) {
+            this.bestKnownHeadBlock = headBlock;
+            this.bestKnownHeadBlockTime = Date.now();
+        }
+    }
+    /**
+     * Check if a node is considered healthy for a given API.
+     */
+    isNodeHealthy(node, api) {
+        const state = this.health.get(node);
+        if (!state)
+            return true; // Unknown nodes are assumed healthy
+        const now = Date.now();
+        // Check overall node health (consecutive failures)
+        if (state.consecutiveFailures >= this.maxFailuresBeforeCooldown) {
+            if (now - state.lastFailure < this.nodeCooldownMs) {
+                return false;
+            }
+        }
+        // Check API-specific health
+        if (api) {
+            const apiState = state.apiFailures.get(api);
+            if (apiState && apiState.count >= this.maxApiFailuresBeforeCooldown) {
+                if (now - apiState.lastFailure < this.apiCooldownMs) {
+                    return false;
+                }
+            }
+        }
+        // Check head block staleness
+        if (state.headBlock > 0 &&
+            this.bestKnownHeadBlock > 0 &&
+            now - state.headBlockUpdatedAt < this.headBlockTtlMs &&
+            now - this.bestKnownHeadBlockTime < this.headBlockTtlMs) {
+            if (this.bestKnownHeadBlock - state.headBlock > this.staleBlockThreshold) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Return nodes ordered by health for a specific API call.
+     * Healthy nodes come first (preserving original order), then unhealthy nodes as fallback.
+     */
+    getOrderedNodes(allNodes, api) {
+        const healthy = [];
+        const unhealthy = [];
+        for (const node of allNodes) {
+            if (this.isNodeHealthy(node, api)) {
+                healthy.push(node);
+            }
+            else {
+                unhealthy.push(node);
+            }
+        }
+        return [...healthy, ...unhealthy];
+    }
+    /**
+     * Reset all health tracking data.
+     */
+    reset() {
+        this.health.clear();
+        this.bestKnownHeadBlock = 0;
+        this.bestKnownHeadBlockTime = 0;
+    }
+    /**
+     * Get a snapshot of current health state for diagnostics.
+     */
+    getHealthSnapshot() {
+        const snapshot = new Map();
+        for (const [node, state] of this.health) {
+            const apiFailures = {};
+            for (const [api, failure] of state.apiFailures) {
+                apiFailures[api] = { count: failure.count };
+            }
+            snapshot.set(node, {
+                consecutiveFailures: state.consecutiveFailures,
+                headBlock: state.headBlock,
+                apiFailures,
+                healthy: this.isNodeHealthy(node),
+            });
+        }
+        return snapshot;
+    }
+}
+exports.NodeHealthTracker = NodeHealthTracker;

package/lib/index.d.ts

@@ -34,6 +34,7 @@
  */
 import * as utils from './utils';
 export { utils };
+export { NodeHealthTracker, HealthTrackerOptions } from './health-tracker';
 export * from './helpers/blockchain';
 export * from './helpers/database';
 export * from './helpers/rc';

package/lib/index.js

@@ -39,6 +39,8 @@ function __export(m) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const utils = require("./utils");
 exports.utils = utils;
+var health_tracker_1 = require("./health-tracker");
+exports.NodeHealthTracker = health_tracker_1.NodeHealthTracker;
 __export(require("./helpers/blockchain"));
 __export(require("./helpers/database"));
 __export(require("./helpers/rc"));

package/lib/utils.d.ts

@@ -34,6 +34,20 @@
  */
 /// <reference types="node" />
 import { EventEmitter } from 'events';
+import { NodeHealthTracker } from './health-tracker';
+/**
+ * Context for smart retry/failover decisions.
+ */
+export interface RetryContext {
+    /** Health tracker instance for per-node, per-API tracking */
+    healthTracker?: NodeHealthTracker;
+    /** The API being called (e.g. "bridge", "condenser_api", "database_api") */
+    api?: string;
+    /** Whether this is a broadcast operation — never retry after request may have been received */
+    isBroadcast?: boolean;
+    /** Whether to log failover events to console */
+    consoleOnFailover?: boolean;
+}
 /**
  * Return a promise that will resove when a specific event is emitted.
  */
@@ -51,9 +65,19 @@ export declare function iteratorStream<T>(iterator: AsyncIterableIterator<T>): N
  */
 export declare function copy<T>(object: T): T;
 /**
- * Fetch API wrapper that retries until timeout is reached.
+ * Smart fetch with immediate failover and per-node health tracking.
+ *
+ * For read operations:
+ * - On failure, immediately try the next healthy node (no backoff within a round)
+ * - After trying all nodes once (one round), apply backoff before the next round
+ * - Stop after failoverThreshold rounds
+ *
+ * For broadcast operations:
+ * - Only retry on pre-connection errors (ECONNREFUSED, ENOTFOUND, etc.)
+ *   where we know the request never reached the server
+ * - NEVER retry after timeout or response errors to prevent double-broadcasting
  */
-export declare function retryingFetch(currentAddress: string, allAddresses: string | string[], opts: any, timeout: number, failoverThreshold: number, consoleOnFailover: boolean, backoff: (tries: number) => number, fetchTimeout?: (tries: number) => number): Promise<{
+export declare function retryingFetch(currentAddress: string, allAddresses: string | string[], opts: any, timeout: number, failoverThreshold: number, consoleOnFailover: boolean, backoff: (tries: number) => number, fetchTimeout?: (tries: number) => number, retryContext?: RetryContext): Promise<{
     response: any;
     currentAddress: string;
 }>;

package/lib/utils.js

@@ -52,8 +52,10 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const cross_fetch_1 = require("cross-fetch");
 const stream_1 = require("stream");
-// TODO: Add more errors that should trigger a failover
-const timeoutErrors = ['timeout', 'ENOTFOUND', 'ECONNREFUSED', 'database lock', 'CERT_HAS_EXPIRED', 'EHOSTUNREACH', 'ECONNRESET', 'ERR_TLS_CERT_ALTNAME_INVALID', 'EAI_AGAIN'];
+// Errors that indicate the request never reached the server — safe to retry even for broadcasts
+const PRE_CONNECTION_ERRORS = ['ECONNREFUSED', 'ENOTFOUND', 'EHOSTUNREACH', 'EAI_AGAIN'];
+// All errors that should trigger failover for read operations
+const FAILOVER_ERRORS = [...PRE_CONNECTION_ERRORS, 'timeout', 'database lock', 'CERT_HAS_EXPIRED', 'ECONNRESET', 'ERR_TLS_CERT_ALTNAME_INVALID', 'ETIMEDOUT', 'EPIPE', 'EPROTO'];
 /**
  * Return a promise that will resove when a specific event is emitted.
  */
@@ -114,70 +116,175 @@ function copy(object) {
 }
 exports.copy = copy;
 /**
- * Fetch API wrapper that retries until timeout is reached.
+ * Check if an error code indicates the request never reached the server.
  */
-function retryingFetch(currentAddress, allAddresses, opts, timeout, failoverThreshold, consoleOnFailover, backoff, fetchTimeout) {
+function isPreConnectionError(error) {
+    if (!error || !error.code)
+        return false;
+    return PRE_CONNECTION_ERRORS.some((code) => error.code.includes(code));
+}
+/**
+ * Check if an error should trigger failover for read operations.
+ * Matches any known network/timeout error, or errors with no code (HTTP errors).
+ */
+function shouldFailover(error) {
+    if (!error)
+        return true;
+    // HTTP errors (from !response.ok) have no .code — they should trigger failover
+    if (!error.code)
+        return true;
+    return FAILOVER_ERRORS.some((code) => error.code.includes(code));
+}
+/**
+ * Get the next node in the ordered list (wraps around).
+ */
+function nextNode(nodes, currentIndex) {
+    return (currentIndex + 1) % nodes.length;
+}
+/**
+ * Smart fetch with immediate failover and per-node health tracking.
+ *
+ * For read operations:
+ * - On failure, immediately try the next healthy node (no backoff within a round)
+ * - After trying all nodes once (one round), apply backoff before the next round
+ * - Stop after failoverThreshold rounds
+ *
+ * For broadcast operations:
+ * - Only retry on pre-connection errors (ECONNREFUSED, ENOTFOUND, etc.)
+ *   where we know the request never reached the server
+ * - NEVER retry after timeout or response errors to prevent double-broadcasting
+ */
+function retryingFetch(currentAddress, allAddresses, opts, timeout, failoverThreshold, consoleOnFailover, backoff, fetchTimeout, retryContext) {
+    var _a;
     return __awaiter(this, void 0, void 0, function* () {
-        let start = Date.now();
-        let tries = 0;
+        const { healthTracker, api, isBroadcast } = retryContext || {};
+        const logFailover = (_a = retryContext === null || retryContext === void 0 ? void 0 : retryContext.consoleOnFailover) !== null && _a !== void 0 ? _a : consoleOnFailover;
+        // Build ordered node list: healthy nodes first, then unhealthy as fallback
+        let orderedNodes;
+        if (Array.isArray(allAddresses) && allAddresses.length > 1) {
+            orderedNodes = healthTracker
+                ? healthTracker.getOrderedNodes(allAddresses, api)
+                : [...allAddresses];
+        }
+        else {
+            orderedNodes = Array.isArray(allAddresses) ? allAddresses : [allAddresses];
+        }
+        // Always start from the healthiest node (index 0 of the ordered list).
+        // The health tracker already sorted nodes with healthy ones first,
+        // so starting from 0 ensures we use the best available node.
+        let nodeIndex = 0;
+        const totalNodes = orderedNodes.length;
+        const startTime = Date.now();
+        let nodesTriedInRound = 0;
         let round = 0;
-        do {
+        let lastError;
+        // tslint:disable-next-line: no-constant-condition
+        while (true) {
+            const node = orderedNodes[nodeIndex];
             try {
                 if (fetchTimeout) {
-                    opts.timeout = fetchTimeout(tries);
+                    opts.timeout = fetchTimeout(nodesTriedInRound);
                 }
-                const response = yield cross_fetch_1.default(currentAddress, opts);
+                const response = yield cross_fetch_1.default(node, opts);
                 if (!response.ok) {
+                    // Support for Drone: HTTP 500 with valid JSON-RPC response
+                    if (response.status === 500) {
+                        try {
+                            const resJson = yield response.json();
+                            if (resJson.jsonrpc === '2.0') {
+                                if (healthTracker && api)
+                                    healthTracker.recordSuccess(node, api);
+                                return { response: resJson, currentAddress: node };
+                            }
+                        }
+                        catch (_b) {
+                            // JSON parse failed, fall through to error handling
+                        }
+                    }
                     throw new Error(`HTTP ${response.status}: ${response.statusText}`);
                 }
-                return { response: yield response.json(), currentAddress };
+                const responseJson = yield response.json();
+                // Record success in health tracker
+                if (healthTracker && api) {
+                    healthTracker.recordSuccess(node, api);
+                }
+                return { response: responseJson, currentAddress: node };
             }
             catch (error) {
-                if (timeout !== 0 && Date.now() - start > timeout) {
-                    if ((!error || !error.code) && Array.isArray(allAddresses)) {
-                        // If error is empty or not code is present, it means rpc is down => switch
-                        currentAddress = failover(currentAddress, allAddresses, currentAddress, consoleOnFailover);
+                lastError = error;
+                // Record failure in health tracker
+                if (healthTracker && api) {
+                    healthTracker.recordFailure(node, api);
+                }
+                // === BROADCAST SAFETY ===
+                // For broadcasts, only retry if the request definitely never reached the server.
+                // If there's any chance the server received it, throw immediately to prevent
+                // double-broadcasting (e.g. double transfers, double votes).
+                if (isBroadcast) {
+                    if (isPreConnectionError(error) && totalNodes > 1) {
+                        // Safe to try another node — request never left the client
+                        nodeIndex = nextNode(orderedNodes, nodeIndex);
+                        nodesTriedInRound++;
+                        if (nodesTriedInRound >= totalNodes) {
+                            // Tried all nodes, give up for broadcasts
+                            throw error;
+                        }
+                        if (logFailover) {
+                            // tslint:disable-next-line: no-console
+                            console.log(`Broadcast failover to: ${orderedNodes[nodeIndex]} (${error.code}, request never sent)`);
+                        }
+                        continue;
                     }
-                    else {
-                        const isFailoverError = timeoutErrors.filter((fe) => error && error.code && error.code.includes(fe)).length > 0;
-                        if (isFailoverError &&
-                            Array.isArray(allAddresses) &&
-                            allAddresses.length > 1) {
-                            if (round < failoverThreshold) {
-                                start = Date.now();
-                                tries = -1;
-                                if (failoverThreshold > 0) {
-                                    round++;
-                                }
-                                currentAddress = failover(currentAddress, allAddresses, currentAddress, consoleOnFailover);
-                            }
-                            else {
-                                error.message = `[${error.code}] tried ${failoverThreshold} times with ${allAddresses.join(',')}`;
+                    // Timeout, HTTP error, or unknown error — request may have been received.
+                    // Do NOT retry. Throw immediately.
+                    throw error;
+                }
+                // === READ OPERATION FAILOVER ===
+                if (!shouldFailover(error)) {
+                    // Unrecognized error type — don't failover, throw immediately
+                    throw error;
+                }
+                // Try next node immediately (no backoff within a round)
+                if (totalNodes > 1) {
+                    nodeIndex = nextNode(orderedNodes, nodeIndex);
+                    nodesTriedInRound++;
+                    if (nodesTriedInRound >= totalNodes) {
+                        // Completed a full round through all nodes
+                        nodesTriedInRound = 0;
+                        // failoverThreshold=0 means retry forever (only timeout can stop it)
+                        if (failoverThreshold > 0) {
+                            round++;
+                            if (round >= failoverThreshold) {
+                                error.message = `All ${totalNodes} nodes failed after ${failoverThreshold} rounds. ` +
+                                    `Last error: [${error.code || 'HTTP'}] ${error.message}. ` +
+                                    `Nodes: ${orderedNodes.join(', ')}`;
                                 throw error;
                             }
                         }
-                        else {
-                            // tslint:disable-next-line: no-console
-                            console.error(`Didn't failover for error ${error.code ? 'code' : 'message'}: [${error.code || error.message}]`);
+                        // Check total timeout before starting next round
+                        if (timeout !== 0 && Date.now() - startTime > timeout) {
                             throw error;
                         }
+                        // Backoff between rounds (not between individual node attempts)
+                        yield sleep(backoff(round));
+                    }
+                    if (logFailover) {
+                        // tslint:disable-next-line: no-console
+                        console.log(`Switched Hive RPC: ${orderedNodes[nodeIndex]} (previous: ${node}, error: ${error.code || error.message})`);
+                    }
+                }
+                else {
+                    // Single node: use backoff and retry same node (legacy behavior)
+                    if (timeout !== 0 && Date.now() - startTime > timeout) {
+                        throw error;
                     }
+                    yield sleep(backoff(nodesTriedInRound++));
                 }
-                yield sleep(backoff(tries++));
             }
-        } while (true);
+        }
     });
 }
 exports.retryingFetch = retryingFetch;
-const failover = (url, urls, currentAddress, consoleOnFailover) => {
-    const index = urls.indexOf(url);
-    const targetUrl = urls.length === index + 1 ? urls[0] : urls[index + 1];
-    if (consoleOnFailover) {
-        // tslint:disable-next-line: no-console
-        console.log(`Switched Hive RPC: ${targetUrl} (previous: ${currentAddress})`);
-    }
-    return targetUrl;
-};
 // Hack to be able to generate a valid witness_set_properties op
 // Can hopefully be removed when hived's JSON representation is fixed
 const ByteBuffer = require("@ecency/bytebuffer");