@hiveio/dhive 1.3.2 → 1.3.3

package/lib/client.d.ts

@@ -40,6 +40,7 @@ import { HivemindAPI } from './helpers/hivemind';
 import { AccountByKeyAPI } from './helpers/key';
 import { RCAPI } from './helpers/rc';
 import { TransactionStatusAPI } from './helpers/transaction';
+import { NodeHealthTracker, HealthTrackerOptions } from './health-tracker';
 /**
  * Library version.
  */
@@ -102,6 +103,11 @@ export interface ClientOptions {
      * Deprecated - don't use
      */
     rebrandedApi?: boolean;
+    /**
+     * Options for the node health tracker.
+     * Controls cooldown periods, stale block thresholds, etc.
+     */
+    healthTrackerOptions?: HealthTrackerOptions;
 }
 /**
  * RPC Client
@@ -146,6 +152,11 @@ export declare class Client {
      * Transaction status API helper.
      */
     readonly transaction: TransactionStatusAPI;
+    /**
+     * Node health tracker for smart failover.
+     * Tracks per-node, per-API health and head block freshness.
+     */
+    readonly healthTracker: NodeHealthTracker;
     /**
      * Chain ID for current network.
      */

package/lib/client.js

@@ -53,6 +53,7 @@ const hivemind_1 = require("./helpers/hivemind");
 const key_1 = require("./helpers/key");
 const rc_1 = require("./helpers/rc");
 const transaction_1 = require("./helpers/transaction");
+const health_tracker_1 = require("./health-tracker");
 const utils_1 = require("./utils");
 /**
  * Library version.
@@ -94,6 +95,7 @@ class Client {
         this.backoff = options.backoff || defaultBackoff;
         this.failoverThreshold = options.failoverThreshold || 3;
         this.consoleOnFailover = options.consoleOnFailover || false;
+        this.healthTracker = new health_tracker_1.NodeHealthTracker(options.healthTrackerOptions);
         this.database = new database_1.DatabaseAPI(this);
         this.broadcast = new broadcast_1.BroadcastAPI(this);
         this.blockchain = new blockchain_1.Blockchain(this);
@@ -125,6 +127,8 @@ class Client {
      */
     call(api, method, params = []) {
         return __awaiter(this, void 0, void 0, function* () {
+            const isBroadcast = api === 'network_broadcast_api' ||
+                method.startsWith('broadcast_transaction');
             const request = {
                 id: 0,
                 jsonrpc: '2.0',
@@ -159,18 +163,32 @@ class Client {
                 opts.agent = this.options.agent;
             }
             let fetchTimeout;
-            if (api !== 'network_broadcast_api' &&
-                !method.startsWith('broadcast_transaction')) {
+            if (!isBroadcast) {
                 // bit of a hack to work around some nodes high error rates
                 // only effective in node.js (until timeout spec lands in browsers)
                 fetchTimeout = (tries) => (tries + 1) * 500;
             }
-            const { response, currentAddress } = yield utils_1.retryingFetch(this.currentAddress, this.address, opts, this.timeout, this.failoverThreshold, this.consoleOnFailover, this.backoff, fetchTimeout);
+            const { response, currentAddress } = yield utils_1.retryingFetch(this.currentAddress, this.address, opts, this.timeout, this.failoverThreshold, this.consoleOnFailover, this.backoff, fetchTimeout, {
+                healthTracker: this.healthTracker,
+                api,
+                isBroadcast,
+                consoleOnFailover: this.consoleOnFailover,
+            });
             // After failover, change the currently active address
             if (currentAddress !== this.currentAddress) {
                 this.currentAddress = currentAddress;
             }
-            // resolve FC error messages into something more readable
+            // Passively track head block from get_dynamic_global_properties responses.
+            // This costs nothing — we just inspect data we already fetched.
+            if (response.result &&
+                method === 'get_dynamic_global_properties' &&
+                response.result.head_block_number) {
+                this.healthTracker.updateHeadBlock(currentAddress, response.result.head_block_number);
+            }
+            // Handle RPC-level errors.
+            // Unlike network errors, these mean the node responded but returned an error.
+            // We record it as an API-specific failure so the health tracker can
+            // deprioritize this node for this API in future calls.
             if (response.error) {
                 const formatValue = (value) => {
                     switch (typeof value) {
@@ -200,6 +218,17 @@ class Client {
                         message += ' ' + unformattedData.join(' ');
                     }
                 }
+                // Track RPC errors that indicate node/plugin issues (not user errors).
+                // JSON-RPC error codes (response.error.code):
+                //   -32601 = Method not found (plugin not enabled on this node)
+                //   -32603 = Internal error (node issue)
+                //   -32003 = Hive assertion error (user error — bad params, invalid account)
+                // Only API/plugin errors should be tracked, and only as API-specific failures
+                // (not global node failures) since other APIs on this node may work fine.
+                const rpcCode = response.error.code;
+                if (rpcCode === -32601 || rpcCode === -32603) {
+                    this.healthTracker.recordApiFailure(currentAddress, api);
+                }
                 throw new verror_1.VError({ info: data, name: 'RPCError' }, message);
             }
             assert.equal(response.id, request.id, 'got invalid response id');

package/lib/health-tracker.d.ts

@@ -0,0 +1,100 @@
+/**
+ * @file Node health tracking for smart failover.
+ * @license BSD-3-Clause-No-Military-License
+ *
+ * Tracks per-node, per-API health to enable intelligent failover decisions.
+ * Nodes that fail for specific APIs are deprioritized for those APIs while
+ * remaining available for others. Stale nodes (behind on head block) are
+ * also deprioritized.
+ */
+export interface HealthTrackerOptions {
+    /**
+     * How long (ms) to deprioritize a node after consecutive failures.
+     * Default: 30 seconds.
+     */
+    nodeCooldownMs?: number;
+    /**
+     * How long (ms) to deprioritize a node for a specific API after failures.
+     * Default: 60 seconds.
+     */
+    apiCooldownMs?: number;
+    /**
+     * Number of consecutive failures before a node enters cooldown.
+     * Default: 3.
+     */
+    maxFailuresBeforeCooldown?: number;
+    /**
+     * Number of API-specific failures before deprioritizing for that API.
+     * Default: 2.
+     */
+    maxApiFailuresBeforeCooldown?: number;
+    /**
+     * How many blocks behind the best known head block a node can be
+     * before being considered stale. Default: 30.
+     */
+    staleBlockThreshold?: number;
+    /**
+     * How long (ms) head block data remains valid for staleness checks.
+     * Default: 2 minutes.
+     */
+    headBlockTtlMs?: number;
+}
+export declare class NodeHealthTracker {
+    private health;
+    private bestKnownHeadBlock;
+    private bestKnownHeadBlockTime;
+    private readonly nodeCooldownMs;
+    private readonly apiCooldownMs;
+    private readonly maxFailuresBeforeCooldown;
+    private readonly maxApiFailuresBeforeCooldown;
+    private readonly staleBlockThreshold;
+    private readonly headBlockTtlMs;
+    constructor(options?: HealthTrackerOptions);
+    private getOrCreate;
+    /**
+     * Record a successful call to a node for a specific API.
+     * Clears consecutive failure counter and API-specific failures for this API.
+     */
+    recordSuccess(node: string, api: string): void;
+    /**
+     * Record a network-level failure (timeout, connection refused, HTTP error).
+     * Increments both the global consecutive failure counter and the API-specific counter.
+     */
+    recordFailure(node: string, api: string): void;
+    /**
+     * Record an API/plugin-specific failure (e.g. "method not found", "plugin not enabled").
+     * Only increments the per-API counter, NOT the global consecutive failure counter.
+     * This prevents a node with a disabled plugin from being penalized for all APIs.
+     */
+    recordApiFailure(node: string, api: string): void;
+    private incrementApiFailure;
+    /**
+     * Update head block number for a node.
+     * Called passively when get_dynamic_global_properties responses are observed.
+     */
+    updateHeadBlock(node: string, headBlock: number): void;
+    /**
+     * Check if a node is considered healthy for a given API.
+     */
+    isNodeHealthy(node: string, api?: string): boolean;
+    /**
+     * Return nodes ordered by health for a specific API call.
+     * Healthy nodes come first (preserving original order), then unhealthy nodes as fallback.
+     */
+    getOrderedNodes(allNodes: string[], api?: string): string[];
+    /**
+     * Reset all health tracking data.
+     */
+    reset(): void;
+    /**
+     * Get a snapshot of current health state for diagnostics.
+     */
+    getHealthSnapshot(): Map<string, {
+        consecutiveFailures: number;
+        headBlock: number;
+        apiFailures: Record<string, {
+            count: number;
+        }>;
+        healthy: boolean;
+    }>;
+}

package/lib/health-tracker.js

@@ -0,0 +1,167 @@
+"use strict";
+/**
+ * @file Node health tracking for smart failover.
+ * @license BSD-3-Clause-No-Military-License
+ *
+ * Tracks per-node, per-API health to enable intelligent failover decisions.
+ * Nodes that fail for specific APIs are deprioritized for those APIs while
+ * remaining available for others. Stale nodes (behind on head block) are
+ * also deprioritized.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+class NodeHealthTracker {
+    constructor(options = {}) {
+        var _a, _b, _c, _d, _e, _f;
+        this.health = new Map();
+        this.bestKnownHeadBlock = 0;
+        this.bestKnownHeadBlockTime = 0;
+        this.nodeCooldownMs = (_a = options.nodeCooldownMs) !== null && _a !== void 0 ? _a : 30000;
+        this.apiCooldownMs = (_b = options.apiCooldownMs) !== null && _b !== void 0 ? _b : 60000;
+        this.maxFailuresBeforeCooldown = (_c = options.maxFailuresBeforeCooldown) !== null && _c !== void 0 ? _c : 3;
+        this.maxApiFailuresBeforeCooldown = (_d = options.maxApiFailuresBeforeCooldown) !== null && _d !== void 0 ? _d : 2;
+        this.staleBlockThreshold = (_e = options.staleBlockThreshold) !== null && _e !== void 0 ? _e : 30;
+        this.headBlockTtlMs = (_f = options.headBlockTtlMs) !== null && _f !== void 0 ? _f : 120000;
+    }
+    getOrCreate(node) {
+        let state = this.health.get(node);
+        if (!state) {
+            state = {
+                apiFailures: new Map(),
+                consecutiveFailures: 0,
+                lastFailure: 0,
+                headBlock: 0,
+                headBlockUpdatedAt: 0,
+            };
+            this.health.set(node, state);
+        }
+        return state;
+    }
+    /**
+     * Record a successful call to a node for a specific API.
+     * Clears consecutive failure counter and API-specific failures for this API.
+     */
+    recordSuccess(node, api) {
+        const state = this.getOrCreate(node);
+        state.consecutiveFailures = 0;
+        state.apiFailures.delete(api);
+    }
+    /**
+     * Record a network-level failure (timeout, connection refused, HTTP error).
+     * Increments both the global consecutive failure counter and the API-specific counter.
+     */
+    recordFailure(node, api) {
+        const state = this.getOrCreate(node);
+        state.consecutiveFailures++;
+        state.lastFailure = Date.now();
+        this.incrementApiFailure(state, api);
+    }
+    /**
+     * Record an API/plugin-specific failure (e.g. "method not found", "plugin not enabled").
+     * Only increments the per-API counter, NOT the global consecutive failure counter.
+     * This prevents a node with a disabled plugin from being penalized for all APIs.
+     */
+    recordApiFailure(node, api) {
+        const state = this.getOrCreate(node);
+        this.incrementApiFailure(state, api);
+    }
+    incrementApiFailure(state, api) {
+        const apiState = state.apiFailures.get(api) || { count: 0, lastFailure: 0 };
+        apiState.count++;
+        apiState.lastFailure = Date.now();
+        state.apiFailures.set(api, apiState);
+    }
+    /**
+     * Update head block number for a node.
+     * Called passively when get_dynamic_global_properties responses are observed.
+     */
+    updateHeadBlock(node, headBlock) {
+        if (!headBlock || headBlock <= 0)
+            return;
+        const state = this.getOrCreate(node);
+        state.headBlock = headBlock;
+        state.headBlockUpdatedAt = Date.now();
+        if (headBlock > this.bestKnownHeadBlock) {
+            this.bestKnownHeadBlock = headBlock;
+            this.bestKnownHeadBlockTime = Date.now();
+        }
+    }
+    /**
+     * Check if a node is considered healthy for a given API.
+     */
+    isNodeHealthy(node, api) {
+        const state = this.health.get(node);
+        if (!state)
+            return true; // Unknown nodes are assumed healthy
+        const now = Date.now();
+        // Check overall node health (consecutive failures)
+        if (state.consecutiveFailures >= this.maxFailuresBeforeCooldown) {
+            if (now - state.lastFailure < this.nodeCooldownMs) {
+                return false;
+            }
+        }
+        // Check API-specific health
+        if (api) {
+            const apiState = state.apiFailures.get(api);
+            if (apiState && apiState.count >= this.maxApiFailuresBeforeCooldown) {
+                if (now - apiState.lastFailure < this.apiCooldownMs) {
+                    return false;
+                }
+            }
+        }
+        // Check head block staleness
+        if (state.headBlock > 0 &&
+            this.bestKnownHeadBlock > 0 &&
+            now - state.headBlockUpdatedAt < this.headBlockTtlMs &&
+            now - this.bestKnownHeadBlockTime < this.headBlockTtlMs) {
+            if (this.bestKnownHeadBlock - state.headBlock > this.staleBlockThreshold) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Return nodes ordered by health for a specific API call.
+     * Healthy nodes come first (preserving original order), then unhealthy nodes as fallback.
+     */
+    getOrderedNodes(allNodes, api) {
+        const healthy = [];
+        const unhealthy = [];
+        for (const node of allNodes) {
+            if (this.isNodeHealthy(node, api)) {
+                healthy.push(node);
+            }
+            else {
+                unhealthy.push(node);
+            }
+        }
+        return [...healthy, ...unhealthy];
+    }
+    /**
+     * Reset all health tracking data.
+     */
+    reset() {
+        this.health.clear();
+        this.bestKnownHeadBlock = 0;
+        this.bestKnownHeadBlockTime = 0;
+    }
+    /**
+     * Get a snapshot of current health state for diagnostics.
+     */
+    getHealthSnapshot() {
+        const snapshot = new Map();
+        for (const [node, state] of this.health) {
+            const apiFailures = {};
+            for (const [api, failure] of state.apiFailures) {
+                apiFailures[api] = { count: failure.count };
+            }
+            snapshot.set(node, {
+                consecutiveFailures: state.consecutiveFailures,
+                headBlock: state.headBlock,
+                apiFailures,
+                healthy: this.isNodeHealthy(node),
+            });
+        }
+        return snapshot;
+    }
+}
+exports.NodeHealthTracker = NodeHealthTracker;

package/lib/index.d.ts

@@ -34,6 +34,8 @@
  */
 import * as utils from './utils';
 export { utils };
+export { NodeHealthTracker } from './health-tracker';
+export type { HealthTrackerOptions } from './health-tracker';
 export * from './helpers/blockchain';
 export * from './helpers/database';
 export * from './helpers/rc';

package/lib/index.js

@@ -39,6 +39,8 @@ function __export(m) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const utils = require("./utils");
 exports.utils = utils;
+var health_tracker_1 = require("./health-tracker");
+exports.NodeHealthTracker = health_tracker_1.NodeHealthTracker;
 __export(require("./helpers/blockchain"));
 __export(require("./helpers/database"));
 __export(require("./helpers/rc"));

package/lib/utils.d.ts

@@ -34,6 +34,20 @@
  */
 /// <reference types="node" />
 import { EventEmitter } from 'events';
+import { NodeHealthTracker } from './health-tracker';
+/**
+ * Context for smart retry/failover decisions.
+ */
+export interface RetryContext {
+    /** Health tracker instance for per-node, per-API tracking */
+    healthTracker?: NodeHealthTracker;
+    /** The API being called (e.g. "bridge", "condenser_api", "database_api") */
+    api?: string;
+    /** Whether this is a broadcast operation — never retry after request may have been received */
+    isBroadcast?: boolean;
+    /** Whether to log failover events to console */
+    consoleOnFailover?: boolean;
+}
 /**
  * Return a promise that will resove when a specific event is emitted.
  */
@@ -51,9 +65,19 @@ export declare function iteratorStream<T>(iterator: AsyncIterableIterator<T>): N
  */
 export declare function copy<T>(object: T): T;
 /**
- * Fetch API wrapper that retries until timeout is reached.
+ * Smart fetch with immediate failover and per-node health tracking.
+ *
+ * For read operations:
+ * - On failure, immediately try the next healthy node (no backoff within a round)
+ * - After trying all nodes once (one round), apply backoff before the next round
+ * - Stop after failoverThreshold rounds
+ *
+ * For broadcast operations:
+ * - Only retry on pre-connection errors (ECONNREFUSED, ENOTFOUND, etc.)
+ *   where we know the request never reached the server
+ * - NEVER retry after timeout or response errors to prevent double-broadcasting
  */
-export declare function retryingFetch(currentAddress: string, allAddresses: string | string[], opts: any, timeout: number, failoverThreshold: number, consoleOnFailover: boolean, backoff: (tries: number) => number, fetchTimeout?: (tries: number) => number): Promise<{
+export declare function retryingFetch(currentAddress: string, allAddresses: string | string[], opts: any, timeout: number, failoverThreshold: number, consoleOnFailover: boolean, backoff: (tries: number) => number, fetchTimeout?: (tries: number) => number, retryContext?: RetryContext): Promise<{
     response: any;
     currentAddress: string;
 }>;