dexbot 1.4.6 → 1.4.8

package/dist/credential-daemon.js

@@ -104,13 +104,15 @@ const { assertPrivatePathSecurity, ensureCredentialRuntimeDirSync, getCredential
 const { buildSessionAccountCache, loadDaemonPrivateKey, } = require('./modules/credential_session_cache');
 const { fetchBootstrapPassword } = require('./modules/launcher/credential_bootstrap');
 const { normalizeBootstrapCredential } = require('./modules/launcher/credential_secret');
-const Logger = require('./modules/logger');
+const Logger = require('./modules/logger').default;
 const { ensureDir, safeUnlink } = require('./modules/utils/fs_utils');
 const { Config } = require('./modules/config');
 const daemonLogger = new Logger('credential-daemon');
 // Resolve project root — handles running from dist/ (compiled) vs source
 const { PATHS } = require('./modules/paths');
 const { getErrorMessage } = require('./modules/utils/errors');
+const { sleep } = require('./modules/order/utils/system');
+const { classifyBroadcastFailure } = require('./modules/broadcast_failure');
 // Unix sockets are required; only Unix-like systems are supported
 const RUNTIME_DIR = getCredentialRuntimeDir({ root: PATHS.PROJECT_ROOT });
 const SOCKET_PATH = getCredentialSocketPath({ root: PATHS.PROJECT_ROOT, runtimeDir: RUNTIME_DIR });
@@ -136,7 +138,7 @@ let auditPruneIntervalTimer = null;
 // Signing client cache: key = `${accountName}:${keyFingerprint(wif)}`, value = full signing client + createdAt.
 // Key rotation: loadDaemonPrivateKey re-reads from vault on every call. If the WIF changes the
 // fingerprint changes → cache miss → new signing client created with the current key. No staleness.
-// Cleared on transport reconnect (see broadcastWithRetry). TTL-pruned (30 min) in pruneStaleSigningClients.
+// Cleared on transport reconnect (see broadcastWithDeadline). TTL-pruned (30 min) in pruneStaleSigningClients.
 const signingClientCache = new Map();
 function debugLog(message, err = null) {
     const suffix = err && getErrorMessage(err) ? `: ${getErrorMessage(err)}` : '';
@@ -396,107 +398,366 @@ async function executeOperationsWithClient(client, operations) {
         operation_results: operationResults,
     };
 }
-async function broadcastWithRetry(accountName, privateKey, broadcastFn, nodeUrl = null) {
-    // The inner deadline caps the TOTAL time spent across BOTH retry attempts
-    // so we always reply to the bot well before its outer socket timer
+/**
+ * Build a typed BROADCAST_DEADLINE error describing an uncertain broadcast.
+ * The bot maps this code to BroadcastUncertainError and runs verify-before-
+ * retry (chain read + adoption) instead of re-broadcasting.
+ */
+function buildUncertainError(accountName, startedAt, detail) {
+    const err = new Error(`${DAEMON_CODES.BROADCAST_DEADLINE}:${detail}`);
+    err.code = DAEMON_CODES.BROADCAST_DEADLINE;
+    err.uncertain = true;
+    err.accountName = accountName;
+    err.startedAt = startedAt;
+    err.ageMs = Date.now() - startedAt;
+    return err;
+}
+/**
+ * Serialize daemon broadcasts: the daemon shares ONE _nativeChainClient and
+ * signing-client cache across all socket clients, so concurrent
+ * broadcastWithDeadline calls would stomp each other's node pinning, connect
+ * state, and cache clearing (a pre-existing hazard, widened by per-node
+ * pinning). Broadcasts are queued as a promise chain; non-broadcast requests
+ * are unaffected.
+ *
+ * The broadcast deadline guard (createBroadcastGuard) is created by the
+ * request handler BEFORE the queue wait, so the deadline timer covers queue
+ * wait + broadcast work together: total daemon wall time per request is
+ * capped at CREDENTIAL_DAEMON_INNER_DEADLINE_MS, well inside the bot's
+ * CREDENTIAL_BROADCAST_TIMEOUT_MS outer socket window. A broadcast that is
+ * still queued when the deadline fires aborts via guard.isFired() before it
+ * starts, and a client socket that dies (bot outer timeout destroys its end,
+ * crash, restart) fires the guard too — so a broadcast can never land on
+ * chain AFTER the bot already verified chain absence and re-broadcast the
+ * same operation.
+ */
+let broadcastChain = Promise.resolve();
+function serializeBroadcast(fn, getWork) {
+    const run = broadcastChain.then(fn, fn);
+    // The chain must wait for the WORK to settle, not the deadline-raced
+    // result: a deadline-aborted broadcast keeps running in the background
+    // (the guard only aborts it at its next checkpoint), and starting the
+    // next queued broadcast while that zombie still touches the shared
+    // _nativeChainClient would stomp its node pinning / transport state.
+    // getWork() is consulted AFTER run settles, when the zombie (if any) is
+    // known; the transport disconnect + guard checkpoints terminate it fast,
+    // so the queue wait stays well inside the next request's inner deadline.
+    broadcastChain = Promise.resolve(getWork ? run.then(() => getWork(), () => getWork()) : run)
+        .then(() => undefined, () => undefined);
+    return run;
+}
+/**
+ * Broadcast deadline guard shared between the queue wait and the broadcast
+ * work. The timer starts at request receipt — before the serializeBroadcast
+ * queue wait — so queued broadcasts cannot outlive the bot's outer socket
+ * window. Firing the guard (deadline exceeded, or the requesting socket
+ * died) aborts the queued/in-flight broadcast: the work checks isFired()
+ * before every attempt, so a late broadcast can never land after the bot
+ * already verified chain absence and re-broadcast the operation.
+ */
+function createBroadcastGuard(accountName, startedAt, deadlineMs) {
+    let fired = false;
+    let timer = null;
+    let rejectGuard = null;
+    const promise = new Promise((_, reject) => {
+        rejectGuard = reject;
+        timer = setTimeout(() => {
+            fired = true;
+            reject(buildUncertainError(accountName, startedAt, `inner broadcast deadline ${deadlineMs}ms exceeded`));
+        }, deadlineMs);
+    });
+    // The guard promise is raced only once the serialized broadcast starts;
+    // while the request sits in the queue a deadline rejection would
+    // otherwise surface as an unhandled rejection.
+    promise.catch(() => { });
+    return {
+        isFired: () => fired,
+        promise,
+        // The candidate in play when the guard fires, set by the broadcast
+        // work loop (null until a candidate is attempted, or for a queued
+        // deadline that never started). The request handler echoes it in the
+        // BROADCAST_DEADLINE reply so the bot blames the node that actually
+        // caused the outcome — never its original preference blindly.
+        currentNode: null,
+        fire: (reason) => {
+            if (fired)
+                return;
+            fired = true;
+            if (timer) {
+                clearTimeout(timer);
+                timer = null;
+            }
+            rejectGuard(buildUncertainError(accountName, startedAt, reason));
+        },
+        clearTimer: () => {
+            if (timer) {
+                clearTimeout(timer);
+                timer = null;
+            }
+        }
+    };
+}
+function resolveInnerDeadlineMs() {
+    return Number.isFinite(Number(TIMING?.CREDENTIAL_DAEMON_INNER_DEADLINE_MS))
+        ? Number(TIMING.CREDENTIAL_DAEMON_INNER_DEADLINE_MS)
+        : 20000;
+}
+async function broadcastWithDeadline(accountName, privateKey, broadcastFn, nodeUrl = null, opts = {}) {
+    // Deadline-capped broadcast: each node gets CREDENTIAL_DAEMON_BROADCAST_RETRIES
+    // attempts pinned to it (the transport sweeps ONLY the pinned node), and
+    // only when they ALL fail with failures that provably never reached the
+    // chain (pre-transmit: connection setup, WebSocket not open, frame send
+    // errors) does the daemon report the node failure to the node health
+    // ledger and rotate to the next best node from the health cache. An
+    // uncertain failure — RPC timeout, or the connection dropped while a
+    // response was pending — may have landed, and re-signing it would
+    // duplicate the transaction on chain (each re-signing produces a new
+    // transaction ID), so it is NEVER retried on any node: it is reported as
+    // BROADCAST_DEADLINE so the bot layer verifies chain inclusion before any
+    // re-broadcast (executeWithRetryOnUncertain, startup adoption). The inner
+    // deadline caps the TOTAL wall time across all attempts — pre-transmit
+    // failures are fast (no RPC wait), so retries never hang the bot — and
+    // guarantees we
+    // reply well before the bot's outer socket timer
     // (CREDENTIAL_BROADCAST_TIMEOUT_MS) fires. If we don't reply in time, the
     // bot raises BroadcastUncertainError and enters the recovery path.
     // See: modules/dexbot_credential_client.ts BroadcastUncertainError.
-    const innerDeadlineMs = Number.isFinite(Number(TIMING?.CREDENTIAL_DAEMON_INNER_DEADLINE_MS))
-        ? Number(TIMING.CREDENTIAL_DAEMON_INNER_DEADLINE_MS)
-        : 20000;
-    const startedAt = Date.now();
-    let deadlineTimer = null;
-    const deadlinePromise = new Promise((_, reject) => {
-        deadlineTimer = setTimeout(() => {
-            const err = new Error(`${DAEMON_CODES.BROADCAST_DEADLINE}:inner broadcast deadline ${innerDeadlineMs}ms exceeded`);
-            err.code = DAEMON_CODES.BROADCAST_DEADLINE;
-            err.uncertain = true;
-            err.accountName = accountName;
-            err.startedAt = startedAt;
-            err.ageMs = Date.now() - startedAt;
-            reject(err);
-        }, innerDeadlineMs);
-    });
-    const maxRetries = TIMING?.CREDENTIAL_DAEMON_BROADCAST_RETRIES ?? 2;
+    const innerDeadlineMs = resolveInnerDeadlineMs();
+    const maxRetries = Number.isFinite(Number(TIMING?.CREDENTIAL_DAEMON_BROADCAST_RETRIES))
+        ? Number(TIMING.CREDENTIAL_DAEMON_BROADCAST_RETRIES)
+        : 3;
+    const retryBackoffMs = Number.isFinite(Number(TIMING?.CREDENTIAL_DAEMON_BROADCAST_BACKOFF_MS))
+        ? Number(TIMING.CREDENTIAL_DAEMON_BROADCAST_BACKOFF_MS)
+        : 1000;
+    const startedAt = opts.startedAt ?? Date.now();
+    // The guard covers the queue wait too when the request handler creates
+    // it before serializeBroadcast (deadline timer started at request
+    // receipt); without one it starts here and covers only the broadcast
+    // work. isFired() lets the retry loop abort before any late background
+    // broadcast, and the finally block resets the transport even while the
+    // broadcast is still in flight.
+    const guard = opts.guard ?? createBroadcastGuard(accountName, startedAt, innerDeadlineMs);
+    const deadlinePromise = guard.promise;
     const work = (async () => {
-        // If a specific node URL is requested, override the global node list
-        // for this broadcast so the retry uses a different backend.
-        const effectiveNodeList = nodeUrl
-            ? [nodeUrl]
-            : (_nativeNodeList.length > 0 ? _nativeNodeList : NODE_MANAGEMENT.DEFAULT_NODES);
-        for (let attempt = 1; attempt <= maxRetries; attempt++) {
-            try {
-                if (_nativeChainClient.getStatus() !== 'connected') {
-                    _nativeChainClient.setNodes(effectiveNodeList);
-                    await _nativeChainClient.connect();
-                    // Transport reconnected — dispose all stale signing clients (heap-dump safety)
-                    // then clear the cache so new clients use the fresh transport.
-                    for (const [, entry] of signingClientCache) {
-                        if (typeof entry.signingClient.dispose === 'function') {
-                            try {
-                                entry.signingClient.dispose();
-                            }
-                            catch (_) { }
-                        }
+        // Pin ALL attempts to ONE node at a time: up to `attemptsPerNode`
+        // attempts on the best candidate, and only when those all fail with
+        // provably-untransmitted errors do we report the node failure and
+        // rotate to the next best node.
+        const attemptsPerNode = Math.max(1, maxRetries);
+        let lastErr = null;
+        const exhaustedNodes = [];
+        // Build the ordered candidate list: the bot-supplied nodeUrl is the
+        // PREFERRED node (attempted first), followed by the healthy node set
+        // ordered best-first by the shared health cache. Re-reads the cache
+        // so bot-side blacklists are reflected immediately; daemon-side
+        // blacklists (daemonNodeHealth) are excluded until their cooldown
+        // expires.
+        const buildCandidates = () => {
+            refreshNodeList();
+            const baseList = _nativeNodeList.length > 0 ? _nativeNodeList : NODE_MANAGEMENT.DEFAULT_NODES;
+            const list = nodeUrl ? [nodeUrl, ...baseList.filter((n) => n !== nodeUrl)] : [...baseList];
+            return list.filter((n) => !exhaustedNodes.includes(n) && !daemonNodeHealth.isBlacklisted(n));
+        };
+        // Candidate pool: healthy nodes best-first. When every available node
+        // is daemon-blacklisted (or the health cache is empty/absent), fall
+        // back to the configured defaults as a LAST RESORT — rebuilt the same
+        // way on every rotation, so each rotation still gets its next
+        // best-effort node: failing the broadcast outright would stall the
+        // order until the bot's next cycle, and a stale blacklist (node
+        // recovered inside its cooldown window) would otherwise go unused.
+        // Ordering: never-failed/recovered nodes first, then blacklisted ones
+        // cooldown-soonest-first.
+        let lastResortWarned = false;
+        const buildCandidatePool = () => {
+            const pool = buildCandidates();
+            if (pool.length > 0)
+                return pool;
+            const state = daemonNodeHealth.getState();
+            const fallback = NODE_MANAGEMENT.DEFAULT_NODES
+                .filter((n) => !exhaustedNodes.includes(n))
+                .sort((a, b) => (state.get(a)?.blacklistedUntil ?? -Infinity) - (state.get(b)?.blacklistedUntil ?? -Infinity));
+            if (fallback.length > 0 && !lastResortWarned) {
+                lastResortWarned = true;
+                daemonLogger.warn?.(`[credential-daemon] All healthy candidates blacklisted; falling back to last-resort node(s) (never-failed first, then cooldown-soonest)`);
+            }
+            return fallback;
+        };
+        // candidates[0] is always the next fresh node: every rotation pushes
+        // the exhausted node onto the ledger and rebuilds the pool, so the
+        // pool shrinks until it is empty.
+        let candidates = buildCandidatePool();
+        // The candidate in play when the deadline fires, exposed to the
+        // request handler (via the guard) so the BROADCAST_DEADLINE reply can
+        // name the node that actually caused it — never the bot's original
+        // preference, which may be innocent after daemon-side rotation.
+        if (opts.guard)
+            opts.guard.currentNode = null;
+        while (candidates.length > 0) {
+            const candidate = candidates[0];
+            if (opts.guard)
+                opts.guard.currentNode = candidate;
+            for (let attempt = 1; attempt <= attemptsPerNode; attempt++) {
+                // Abort as soon as the guard fired (inner deadline won the
+                // race, or the requesting socket died): the bot has been told
+                // the outcome is uncertain and may verify + re-create — a
+                // late background broadcast would duplicate the order.
+                if (guard.isFired()) {
+                    throw buildUncertainError(accountName, startedAt, 'inner broadcast deadline exceeded');
+                }
+                if (attempt > 1) {
+                    await sleep(retryBackoffMs);
+                    if (guard.isFired()) {
+                        throw buildUncertainError(accountName, startedAt, 'inner broadcast deadline exceeded');
                     }
-                    signingClientCache.clear();
                 }
-                const fp = keyFingerprint(String(privateKey));
-                const cacheKey = accountName + ':' + fp;
-                let cached = signingClientCache.get(cacheKey);
-                if (!cached) {
-                    // Proactively evict stale entries for the same account (different fingerprint)
-                    // so key rotation never leaves dead entries in the map.
-                    const stalePrefix = accountName + ':';
-                    for (const [k, v] of signingClientCache) {
-                        if (k !== cacheKey && k.startsWith(stalePrefix)) {
-                            if (typeof v.signingClient.dispose === 'function') {
+                let phase = 'connect';
+                try {
+                    const currentNode = _nativeChainClient.transport?.getNodeUrl?.();
+                    if (_nativeChainClient.getStatus() !== 'connected' || currentNode !== candidate) {
+                        phase = 'connect';
+                        // Pin the attempt to the current candidate: the transport
+                        // sweeps ONLY the pinned node, so all attempts genuinely
+                        // use the SAME node before any rotation.
+                        _nativeChainClient.setNodes([candidate]);
+                        await _nativeChainClient.connect();
+                        // The guard may have fired while the connect handshake was
+                        // in flight (slow node + inner deadline). Abort here —
+                        // before any signing-client work — so the sign+broadcast
+                        // never happens after the bot was told BROADCAST_DEADLINE:
+                        // a late background broadcast would duplicate the order.
+                        if (guard.isFired()) {
+                            throw buildUncertainError(accountName, startedAt, 'inner broadcast deadline exceeded during connect');
+                        }
+                        // Transport reconnected — dispose all stale signing clients (heap-dump safety)
+                        // then clear the cache so new clients use the fresh transport.
+                        for (const [, entry] of signingClientCache) {
+                            if (typeof entry.signingClient.dispose === 'function') {
                                 try {
-                                    v.signingClient.dispose();
+                                    entry.signingClient.dispose();
                                 }
                                 catch (_) { }
                             }
-                            signingClientCache.delete(k);
                         }
+                        signingClientCache.clear();
+                    }
+                    const fp = keyFingerprint(String(privateKey));
+                    const cacheKey = accountName + ':' + fp;
+                    let cached = signingClientCache.get(cacheKey);
+                    if (!cached) {
+                        // Proactively evict stale entries for the same account (different fingerprint)
+                        // so key rotation never leaves dead entries in the map.
+                        const stalePrefix = accountName + ':';
+                        for (const [k, v] of signingClientCache) {
+                            if (k !== cacheKey && k.startsWith(stalePrefix)) {
+                                if (typeof v.signingClient.dispose === 'function') {
+                                    try {
+                                        v.signingClient.dispose();
+                                    }
+                                    catch (_) { }
+                                }
+                                signingClientCache.delete(k);
+                            }
+                        }
+                        pruneStaleSigningClients();
+                        // Pre-convert WIF string → Buffer so dispose() can actually zero the bytes.
+                        // The WIF string goes out of scope at function return and is GC'd (non-deterministic),
+                        // but the Buffer passed to createSigningClient is filled with 0 by dispose() on eviction/shutdown.
+                        const { createSigningClient, wifToBuffer } = require('./modules/bitshares-native');
+                        const keyBuffer = wifToBuffer(String(privateKey));
+                        const signingClient = createSigningClient(_nativeChainClient, accountName, keyBuffer);
+                        cached = { signingClient, createdAt: Date.now() };
+                        signingClientCache.set(cacheKey, cached);
+                    }
+                    const client = cached.signingClient.client;
+                    await client.initPromise;
+                    // The guard may have fired while the signing client fetched
+                    // account/order metadata. Abort before the broadcast RPC so
+                    // nothing is transmitted post-deadline — a zombie broadcast
+                    // landing after the bot verified chain absence would duplicate
+                    // the operation.
+                    if (guard.isFired()) {
+                        throw buildUncertainError(accountName, startedAt, 'inner broadcast deadline exceeded during signing client init');
                     }
-                    pruneStaleSigningClients();
-                    // Pre-convert WIF string → Buffer so dispose() can actually zero the bytes.
-                    // The WIF string goes out of scope at function return and is GC'd (non-deterministic),
-                    // but the Buffer passed to createSigningClient is filled with 0 by dispose() on eviction/shutdown.
-                    const { createSigningClient, wifToBuffer } = require('./modules/bitshares-native');
-                    const keyBuffer = wifToBuffer(String(privateKey));
-                    const signingClient = createSigningClient(_nativeChainClient, accountName, keyBuffer);
-                    cached = { signingClient, createdAt: Date.now() };
-                    signingClientCache.set(cacheKey, cached);
+                    phase = 'broadcast';
+                    const result = await broadcastFn(client);
+                    // Success — restore the full node list so other users of
+                    // the shared client don't inherit the single-node pin.
+                    _nativeChainClient.setNodes(_nativeNodeList.length > 0 ? _nativeNodeList : NODE_MANAGEMENT.DEFAULT_NODES);
+                    return result;
                 }
-                const client = cached.signingClient.client;
-                await client.initPromise;
-                return await broadcastFn(client);
-            }
-            catch (err) {
-                if (attempt === maxRetries)
+                catch (err) {
+                    // Guard-fired aborts (deadline during connect/init) must
+                    // propagate as uncertain immediately — they are never
+                    // retryable, even during the connect phase, and a retry
+                    // would just re-enter the same deadlined attempt.
+                    if (err?.code === DAEMON_CODES.BROADCAST_DEADLINE || err?.uncertain === true) {
+                        throw err;
+                    }
+                    // Connect-phase failures are always pre-transmit: no broadcast
+                    // has been attempted yet. Broadcast-phase failures are only
+                    // retryable when the RPC frame provably never reached the wire.
+                    const cls = phase === 'connect' ? 'retryable' : classifyBroadcastFailure(err);
+                    if (cls === 'retryable') {
+                        lastErr = err;
+                        const retryLabel = attempt < attemptsPerNode ? 'retrying same node' : 'exhausted; rotating to next best node';
+                        debugLog(`Broadcast attempt ${attempt}/${attemptsPerNode} on ${candidate} failed pre-transmit (${getErrorMessage(err)}); ${retryLabel}`);
+                        // Reset the transport so the next attempt reconnects
+                        // cleanly against the pinned node.
+                        try {
+                            _nativeChainClient.disconnect();
+                        }
+                        catch (_) { }
+                        continue;
+                    }
+                    if (cls === 'uncertain') {
+                        // The transaction may have landed — never re-sign on ANY
+                        // node. Report the typed uncertain reply so the bot's
+                        // verify-before-retry machinery checks chain inclusion
+                        // before any re-create.
+                        throw buildUncertainError(accountName, startedAt, getErrorMessage(err));
+                    }
+                    // 'definite': the chain rejected the transaction or it could
+                    // not be built — nothing landed; the error propagates as-is.
                     throw err;
-                debugLog(`Broadcast failed (attempt ${attempt}), reconnecting: ${getErrorMessage(err)}`);
-                const staleKey = accountName + ':' + keyFingerprint(String(privateKey));
-                const staleEntry = signingClientCache.get(staleKey);
-                if (staleEntry && typeof staleEntry.signingClient.dispose === 'function') {
-                    staleEntry.signingClient.dispose();
-                }
-                signingClientCache.delete(staleKey);
-                try {
-                    _nativeChainClient.disconnect();
                 }
-                catch (_) { }
             }
+            // All attempts on this candidate failed pre-transmit: nothing ever
+            // reached the chain via this node. Report the failure so the node
+            // health ledger counts it toward blacklisting, then rotate to the
+            // next best node (re-reading the health cache first).
+            exhaustedNodes.push(candidate);
+            daemonNodeHealth.reportFailure(candidate, getErrorMessage(lastErr) || 'pre-transmit broadcast failure');
+            candidates = buildCandidatePool();
+        }
+        // All candidates failed pre-transmit (or none were usable): nothing
+        // ever reached the chain.
+        if (!lastErr) {
+            lastErr = new Error('Broadcast not attempted: no usable node candidates (all blacklisted or unavailable)');
         }
+        throw lastErr;
     })();
+    // Expose the WORK promise (not the deadline-raced result) to the broadcast
+    // queue: serializeBroadcast chains the next queued broadcast on it, so a
+    // deadline-aborted zombie cannot overlap the next broadcast on the shared
+    // transport. The handler reads it back via opts.workRef.
+    if (opts.workRef && typeof opts.workRef === 'object') {
+        opts.workRef.work = work;
+    }
     try {
         return await Promise.race([work, deadlinePromise]);
     }
     finally {
-        if (deadlineTimer)
-            clearTimeout(deadlineTimer);
+        guard.clearTimer();
+        if (guard.isFired()) {
+            // The guard won while the broadcast may still be in flight —
+            // drop the connection so the next request starts clean. The
+            // transport abort also kills any in-flight connect handshake, so
+            // the zombie cannot resurrect a connection afterwards.
+            try {
+                _nativeChainClient.disconnect();
+            }
+            catch (_) { }
+        }
         // Prune stale signing clients on every broadcast, not just on cache
         // miss, so long-running daemons don't accumulate stale entries when
         // accounts always hit the cache (same WIF, no rotation).
@@ -525,6 +786,14 @@ function refreshNodeList() {
         }
     }
 }
+/**
+ * Daemon-local node health ledger: blacklists persistently failing nodes so
+ * the broadcast path rotates away from them and the shared health cache is
+ * updated when a node hits the failure threshold. See
+ * modules/daemon_node_health.ts for semantics.
+ */
+const { createNodeHealthLedger } = require('./modules/daemon_node_health');
+const daemonNodeHealth = createNodeHealthLedger({ logger: daemonLogger });
 function getCredentialDaemonNodeRefreshIntervalMs(settings) {
     const configured = settings?.NODES?.credentialDaemonRefreshIntervalMs
         ?? settings?.NODES?.CREDENTIAL_DAEMON_NODE_REFRESH_INTERVAL_MS
@@ -720,9 +989,22 @@ async function initialize() {
  */
 function handleConnection(socket) {
     let buffer = '';
+    // Broadcast deadline guards for requests on this socket. Fired when the
+    // socket dies (bot outer timeout destroys its end, crash, restart): the
+    // queued/in-flight broadcast for a client that can no longer receive the
+    // reply must abort — otherwise it can land on chain after the bot
+    // verified chain absence and re-broadcast the same operation.
+    const activeGuards = new Set();
+    const abortSocketGuards = () => {
+        for (const guard of activeGuards) {
+            guard.fire('client socket closed before broadcast reply');
+        }
+        activeGuards.clear();
+    };
     socket.setTimeout(TIMING.CREDENTIAL_DAEMON_SOCKET_TIMEOUT_MS);
     socket.on('timeout', () => {
         daemonLogger.debug?.('[credential-daemon] Socket timeout — client idle');
+        abortSocketGuards();
         socket.destroy();
     });
     socket.on('data', (data) => {
@@ -738,7 +1020,7 @@ function handleConnection(socket) {
             buffer = lines.pop() || ''; // Keep incomplete line in buffer
             for (const line of lines) {
                 if (line.trim()) {
-                    processRequest(line.trim(), socket);
+                    processRequest(line.trim(), socket, activeGuards);
                 }
             }
         }
@@ -747,12 +1029,17 @@ function handleConnection(socket) {
         }
     });
     socket.on('end', () => {
+        abortSocketGuards();
         socket.destroy();
     });
     socket.on('error', (error) => {
         daemonLogger.debug?.('[credential-daemon] Socket error: ' + getErrorMessage(error));
+        abortSocketGuards();
         socket.destroy();
     });
+    socket.on('close', () => {
+        abortSocketGuards();
+    });
 }
 /**
  * Process incoming credential request from client.
@@ -762,7 +1049,7 @@ function handleConnection(socket) {
  * @param {string} requestStr - JSON string with {type, accountName}
  * @param {net.Socket} socket - Client socket to send response
  */
-function processRequest(requestStr, socket) {
+function processRequest(requestStr, socket, activeGuards = new Set()) {
     if (daemonShuttingDown)
         return;
     // The outer try/catch handles JSON parse errors and any synchronous throws.
@@ -862,9 +1149,20 @@ function processRequest(requestStr, socket) {
                 }
                 const privateKey = await loadCurrentPrivateKey(accountName);
                 const broadcastNodeUrl = typeof request.nodeUrl === 'string' ? request.nodeUrl : null;
+                // Start the broadcast deadline NOW, before the
+                // serializeBroadcast queue wait, so total daemon wall time
+                // per request stays inside the bot's outer socket window.
+                // If the deadline fires while queued (burst concurrency),
+                // the queued broadcast aborts via guard.isFired() before
+                // it starts — it can never land after the bot verified
+                // chain absence and re-broadcast the same operation.
+                const broadcastStartedAt = Date.now();
+                const broadcastGuard = createBroadcastGuard(accountName, broadcastStartedAt, resolveInnerDeadlineMs());
+                activeGuards.add(broadcastGuard);
+                const broadcastWorkRef = { work: null };
                 let signResult;
                 try {
-                    signResult = await broadcastWithRetry(accountName, privateKey, (client) => client.broadcast(operation), broadcastNodeUrl);
+                    signResult = await serializeBroadcast(() => broadcastWithDeadline(accountName, privateKey, (client) => client.broadcast(operation), broadcastNodeUrl, { guard: broadcastGuard, startedAt: broadcastStartedAt, workRef: broadcastWorkRef }), () => broadcastWorkRef.work || Promise.resolve());
                 }
                 catch (broadcastErr) {
                     if (broadcastErr && broadcastErr.code === DAEMON_CODES.BROADCAST_DEADLINE) {
@@ -872,7 +1170,7 @@ function processRequest(requestStr, socket) {
                             event: 'sign_timeout',
                             accountName,
                             sessionId,
-                            nodeUrl: broadcastNodeUrl,
+                            nodeUrl: broadcastGuard.currentNode || broadcastNodeUrl,
                             opCount: 1,
                             opTypes: [operation && operation.op_name].filter(Boolean),
                             ageMs: broadcastErr.ageMs,
@@ -883,11 +1181,19 @@ function processRequest(requestStr, socket) {
                         });
                         // Tell the bot the chain state is uncertain so it
                         // can run the recovery path (read chain, match by
-                        // fingerprint, adopt or discard).
-                        return sendError(socket, 'chain status uncertain after inner deadline', DAEMON_CODES.BROADCAST_DEADLINE);
+                        // fingerprint, adopt or discard). Name the node
+                        // actually in play at the deadline (the bot's
+                        // preferred node may be innocent after daemon-side
+                        // rotation, and a queued deadline involves none).
+                        return sendError(socket, 'chain status uncertain after inner deadline', DAEMON_CODES.BROADCAST_DEADLINE, broadcastGuard.currentNode
+                            ? { nodeUrl: broadcastGuard.currentNode }
+                            : {});
                     }
                     throw broadcastErr;
                 }
+                finally {
+                    activeGuards.delete(broadcastGuard);
+                }
                 appendAuditLog({
                     event: 'sign_allowed',
                     accountName,
@@ -955,9 +1261,20 @@ function processRequest(requestStr, socket) {
                 }
                 const privateKey = await loadCurrentPrivateKey(accountName);
                 const broadcastNodeUrl = typeof request.nodeUrl === 'string' ? request.nodeUrl : null;
+                // Start the broadcast deadline NOW, before the
+                // serializeBroadcast queue wait, so total daemon wall time
+                // per request stays inside the bot's outer socket window.
+                // If the deadline fires while queued (burst concurrency),
+                // the queued broadcast aborts via guard.isFired() before
+                // it starts — it can never land after the bot verified
+                // chain absence and re-broadcast the same operation.
+                const broadcastStartedAt = Date.now();
+                const broadcastGuard = createBroadcastGuard(accountName, broadcastStartedAt, resolveInnerDeadlineMs());
+                activeGuards.add(broadcastGuard);
+                const broadcastWorkRef = { work: null };
                 let signResult;
                 try {
-                    signResult = await broadcastWithRetry(accountName, privateKey, (client) => executeOperationsWithClient(client, operations), broadcastNodeUrl);
+                    signResult = await serializeBroadcast(() => broadcastWithDeadline(accountName, privateKey, (client) => executeOperationsWithClient(client, operations), broadcastNodeUrl, { guard: broadcastGuard, startedAt: broadcastStartedAt, workRef: broadcastWorkRef }), () => broadcastWorkRef.work || Promise.resolve());
                 }
                 catch (broadcastErr) {
                     if (broadcastErr && broadcastErr.code === DAEMON_CODES.BROADCAST_DEADLINE) {
@@ -965,7 +1282,7 @@ function processRequest(requestStr, socket) {
                             event: 'sign_timeout',
                             accountName,
                             sessionId,
-                            nodeUrl: broadcastNodeUrl,
+                            nodeUrl: broadcastGuard.currentNode || broadcastNodeUrl,
                             opCount: operations.length,
                             opTypes: operations.map((o) => o && o.op_name).filter(Boolean),
                             ageMs: broadcastErr.ageMs,
@@ -974,10 +1291,15 @@ function processRequest(requestStr, socket) {
                                 : null,
                             timestamp: new Date().toISOString(),
                         });
-                        return sendError(socket, 'chain status uncertain after inner deadline', DAEMON_CODES.BROADCAST_DEADLINE);
+                        return sendError(socket, 'chain status uncertain after inner deadline', DAEMON_CODES.BROADCAST_DEADLINE, broadcastGuard.currentNode
+                            ? { nodeUrl: broadcastGuard.currentNode }
+                            : {});
                     }
                     throw broadcastErr;
                 }
+                finally {
+                    activeGuards.delete(broadcastGuard);
+                }
                 appendAuditLog({
                     event: 'sign_allowed',
                     accountName,
@@ -1017,12 +1339,14 @@ function sendSuccess(socket, data) {
  * @param {net.Socket} socket - Client socket
  * @param {string} message - Error message
  * @param {number} code - Error code
+ * @param {Object} [extra] - Extra fields merged into the response (e.g. nodeUrl)
  */
-function sendError(socket, message, code = null) {
+function sendError(socket, message, code = null, extra = {}) {
     const response = JSON.stringify({
         success: false,
         error: message,
-        ...(code ? { code } : {})
+        ...(code ? { code } : {}),
+        ...extra
     });
     socket.write(response + '\n');
     socket.end();