dexbot 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +3 -3
- package/analysis/README.md +33 -13
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/account_resolver.js +5 -6
- package/dist/analysis/analyze_liquidity.js +15 -4
- package/dist/analysis/bot_key_utils.d.ts +44 -1
- package/dist/analysis/bot_key_utils.js +76 -7
- package/dist/analysis/fills_cache.js +2 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -19
- package/dist/analysis/grid_correction_check.js +12 -30
- package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
- package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
- package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
- package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
- package/dist/analysis/liquidity/liquidity_view.js +11 -0
- package/dist/analysis/resolve_bot_accounts.js +7 -6
- package/dist/analysis/trade_profitability.js +2 -1
- package/dist/credential-daemon.js +13 -2
- package/dist/dexbot.js +4 -4
- package/dist/modules/account_bots.d.ts +12 -1
- package/dist/modules/account_bots.js +7 -5
- package/dist/modules/account_orders.d.ts +12 -0
- package/dist/modules/account_orders.js +30 -0
- package/dist/modules/bitshares-native/transport.js +24 -6
- package/dist/modules/bitshares_client.d.ts +24 -6
- package/dist/modules/bitshares_client.js +44 -7
- package/dist/modules/bot_settings.d.ts +2 -0
- package/dist/modules/constants.d.ts +28 -3
- package/dist/modules/constants.js +79 -10
- package/dist/modules/dexbot_class.d.ts +23 -6
- package/dist/modules/dexbot_class.js +44 -0
- package/dist/modules/dexbot_cow_runtime.d.ts +128 -8
- package/dist/modules/dexbot_cow_runtime.js +688 -188
- package/dist/modules/dexbot_fill_runtime.js +124 -35
- package/dist/modules/dexbot_maintenance_runtime.d.ts +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +259 -79
- package/dist/modules/dexbot_startup_runtime.js +32 -20
- package/dist/modules/dexbot_state_recovery.d.ts +33 -1
- package/dist/modules/dexbot_state_recovery.js +167 -5
- package/dist/modules/node_connect_policy.d.ts +8 -1
- package/dist/modules/node_connect_policy.js +18 -2
- package/dist/modules/node_flap_score.d.ts +87 -0
- package/dist/modules/node_flap_score.js +124 -0
- package/dist/modules/node_health_cache.d.ts +40 -3
- package/dist/modules/node_health_cache.js +66 -20
- package/dist/modules/node_manager.d.ts +101 -19
- package/dist/modules/node_manager.js +255 -65
- package/dist/modules/order/accounting.d.ts +9 -4
- package/dist/modules/order/accounting.js +131 -19
- package/dist/modules/order/async_lock.d.ts +7 -1
- package/dist/modules/order/async_lock.js +7 -1
- package/dist/modules/order/export.js +25 -18
- package/dist/modules/order/fill_ledger.d.ts +99 -0
- package/dist/modules/order/fill_ledger.js +103 -0
- package/dist/modules/order/format.d.ts +24 -10
- package/dist/modules/order/format.js +30 -10
- package/dist/modules/order/grid.d.ts +75 -18
- package/dist/modules/order/grid.js +289 -72
- package/dist/modules/order/grid_reconcile.d.ts +5 -1
- package/dist/modules/order/grid_reconcile.js +333 -210
- package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
- package/dist/modules/order/grid_reconcile_internal.js +223 -91
- package/dist/modules/order/index.d.ts +18 -5
- package/dist/modules/order/index.js +5 -5
- package/dist/modules/order/lifecycle.d.ts +353 -0
- package/dist/modules/order/lifecycle.js +1109 -0
- package/dist/modules/order/manager.d.ts +277 -20
- package/dist/modules/order/manager.js +680 -64
- package/dist/modules/order/placement_lock.d.ts +102 -0
- package/dist/modules/order/placement_lock.js +222 -0
- package/dist/modules/order/strategy.js +13 -1
- package/dist/modules/order/sync_engine.d.ts +74 -114
- package/dist/modules/order/sync_engine.js +254 -198
- package/dist/modules/order/utils/math.d.ts +29 -4
- package/dist/modules/order/utils/math.js +61 -4
- package/dist/modules/order/utils/order.d.ts +106 -37
- package/dist/modules/order/utils/order.js +256 -81
- package/dist/modules/order/utils/system.d.ts +99 -6
- package/dist/modules/order/utils/system.js +249 -35
- package/dist/modules/order/utils/validate.d.ts +7 -1
- package/dist/modules/order/utils/validate.js +48 -2
- package/dist/modules/outcome.d.ts +54 -0
- package/dist/modules/outcome.js +51 -0
- package/dist/modules/settings_merge.d.ts +4 -0
- package/dist/modules/settings_merge.js +9 -0
- package/dist/modules/timing_policy.d.ts +82 -0
- package/dist/modules/timing_policy.js +244 -0
- package/dist/modules/types.d.ts +149 -22
- package/dist/scripts/analyze-credit.js +54 -17
- package/dist/scripts/analyze-orders.d.ts +4 -2
- package/dist/scripts/analyze-orders.js +113 -34
- package/dist/scripts/chart_targets.js +11 -15
- package/dist/scripts/test-credit-renewal.js +12 -5
- package/docs/COW_INVARIANTS.md +71 -8
- package/docs/DEXBOT_COMPARISON.md +6 -6
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_PRICE_INVARIANT.md +1 -1
- package/docs/GRID_RECONCILE.md +18 -13
- package/docs/LOGGING.md +25 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +183 -26
- package/docs/README.md +23 -2
- package/docs/WORKFLOW.md +9 -3
- package/docs/architecture.md +68 -13
- package/docs/developer_guide.md +25 -8
- package/modules/README.md +6 -0
- package/package.json +3 -3
- package/scripts/README.md +16 -0
- package/scripts/pdev.sh +14 -1
- package/scripts/pmain.sh +23 -8
- package/scripts/ptest.sh +7 -0
- package/tests/README.md +30 -1
|
@@ -15,7 +15,9 @@
|
|
|
15
15
|
* - Automatic blacklisting of failed nodes
|
|
16
16
|
*
|
|
17
17
|
* 2. Intelligent Node Selection
|
|
18
|
-
* -
|
|
18
|
+
* - Ranks nodes by measured latency plus a time-decayed connection-churn
|
|
19
|
+
* penalty (see modules/node_flap_score.ts), so a fast node that keeps
|
|
20
|
+
* dropping its socket loses to a stable slightly-slower one
|
|
19
21
|
* - Falls back to slow nodes if no healthy ones available
|
|
20
22
|
* - Respects user-preferred node if healthy
|
|
21
23
|
*
|
|
@@ -44,19 +46,49 @@ const require = createRequire(import.meta.url);
|
|
|
44
46
|
import Logger from './order/logger.js';
|
|
45
47
|
import { NODE_MANAGEMENT } from './constants.js';
|
|
46
48
|
import { PATHS, getNodeBlacklistFile } from './paths.js';
|
|
47
|
-
import { compareNodeHealth } from './node_health_cache.js';
|
|
48
49
|
import { writeJsonFileAtomic } from './bots_file_lock.js';
|
|
49
50
|
import { getStorage } from './storage/index.js';
|
|
50
51
|
const storage = getStorage();
|
|
51
52
|
const { readJSON } = storage;
|
|
52
53
|
import { createFailureLedger } from './node_failure_ledger.js';
|
|
53
54
|
const _WebSocket = globalThis.WebSocket;
|
|
54
|
-
import { resolveHealthCacheFile, writeHealthCache, } from './node_health_cache.js';
|
|
55
|
+
import { readHealthCache, resolveHealthCacheFile, writeHealthCache, } from './node_health_cache.js';
|
|
56
|
+
import { dropCounters, effectiveLatencyMs, pruneDropTimes, recordDrop, } from './node_flap_score.js';
|
|
55
57
|
import { getErrorMessage, getErrorCode } from './utils/errors.js';
|
|
56
58
|
import { nowIso } from './order/utils/system.js';
|
|
59
|
+
/**
|
|
60
|
+
* Failure sources that count as a live-operation reliability event for churn
|
|
61
|
+
* scoring/selection.
|
|
62
|
+
*
|
|
63
|
+
* Included:
|
|
64
|
+
* - `connection` an unexpected socket close (code 1006/1001 etc.)
|
|
65
|
+
* - `keep-alive` the socket stayed open but stopped answering
|
|
66
|
+
* - `broadcast` a signed batch was accepted but never confirmed in time
|
|
67
|
+
* (BROADCAST_DEADLINE) — the socket is up, the node is not
|
|
68
|
+
* serving us, which is exactly the instability selection
|
|
69
|
+
* should penalize
|
|
70
|
+
*
|
|
71
|
+
* Excluded:
|
|
72
|
+
* - `health-check` probe-side noise; it already moves the node out of the
|
|
73
|
+
* healthy set, and probes may fail for reasons that never
|
|
74
|
+
* affect a live session
|
|
75
|
+
* - `forced-reconnect` a client-side recovery attempt, not evidence that the
|
|
76
|
+
* node is bad (same rule as shouldCountNodeStrike in
|
|
77
|
+
* bitshares_client)
|
|
78
|
+
*/
|
|
79
|
+
const CHURN_SOURCES = new Set(['connection', 'keep-alive', 'broadcast']);
|
|
80
|
+
/** Positive finite number or the fallback (config may carry 0/NaN/strings). */
|
|
81
|
+
function positiveOr(value, fallback) {
|
|
82
|
+
const n = Number(value);
|
|
83
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
84
|
+
}
|
|
57
85
|
/**
|
|
58
86
|
* NodeManager - Manages health checking and selection of BitShares nodes
|
|
59
87
|
*
|
|
88
|
+
* Selection ranks healthy nodes on `latency + decayed churn penalty`, keeps a
|
|
89
|
+
* time-decaying strike count for blacklisting, and exposes per-node churn
|
|
90
|
+
* counters (drops/h) for logging and the shared health cache.
|
|
91
|
+
*
|
|
60
92
|
* @class
|
|
61
93
|
* @param {Object} config - Configuration object
|
|
62
94
|
* @param {string[]} config.list - List of node URLs to monitor (e.g., wss://dex.iobanker.com/ws)
|
|
@@ -65,8 +97,12 @@ import { nowIso } from './order/utils/system.js';
|
|
|
65
97
|
* @param {number} [config.healthCheck.timeoutMs=5000] - Timeout for each health check
|
|
66
98
|
* @param {number} [config.healthCheck.maxPingMs=3000] - Max acceptable latency
|
|
67
99
|
* @param {number} [config.healthCheck.blacklistThreshold=3] - Failures before blacklist
|
|
100
|
+
* @param {number} [config.healthCheck.strikeDecayMs] - Lifetime of a strike without a new failure
|
|
101
|
+
* @param {number} [config.healthCheck.flapHalfLifeMs] - Half-life of a connection-drop penalty
|
|
102
|
+
* @param {number} [config.healthCheck.flapPenaltyMsPerDrop] - Effective-latency cost per fresh drop
|
|
103
|
+
* @param {number} [config.healthCheck.flapPenaltyMaxMs] - Cap on the decayed churn penalty
|
|
68
104
|
* @param {Object} [config.selection] - Node selection settings
|
|
69
|
-
* @param {string} [config.selection.strategy="latency"] - Selection strategy
|
|
105
|
+
* @param {string} [config.selection.strategy="latency"] - Selection strategy (latency + churn penalty)
|
|
70
106
|
* @param {string} [config.selection.preferredNode] - Optional preferred node (if healthy)
|
|
71
107
|
*/
|
|
72
108
|
class NodeManager {
|
|
@@ -98,9 +134,10 @@ class NodeManager {
|
|
|
98
134
|
timeoutMs: config.healthCheck?.timeoutMs ?? NODE_MANAGEMENT.HEALTH_CHECK_TIMEOUT_MS,
|
|
99
135
|
maxPingMs: config.healthCheck?.maxPingMs ?? NODE_MANAGEMENT.MAX_PING_MS,
|
|
100
136
|
blacklistThreshold: config.healthCheck?.blacklistThreshold ?? NODE_MANAGEMENT.BLACKLIST_THRESHOLD,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
137
|
+
strikeDecayMs: positiveOr(config.healthCheck?.strikeDecayMs, NODE_MANAGEMENT.FAILURE_STRIKE_DECAY_MS),
|
|
138
|
+
flapHalfLifeMs: positiveOr(config.healthCheck?.flapHalfLifeMs, NODE_MANAGEMENT.FLAP_HALF_LIFE_MS),
|
|
139
|
+
flapPenaltyMsPerDrop: positiveOr(config.healthCheck?.flapPenaltyMsPerDrop, NODE_MANAGEMENT.FLAP_PENALTY_MS_PER_DROP),
|
|
140
|
+
flapPenaltyMaxMs: positiveOr(config.healthCheck?.flapPenaltyMaxMs, NODE_MANAGEMENT.FLAP_PENALTY_MAX_MS),
|
|
104
141
|
enabled: config.healthCheck?.enabled ?? true
|
|
105
142
|
},
|
|
106
143
|
selection: {
|
|
@@ -127,6 +164,7 @@ class NodeManager {
|
|
|
127
164
|
});
|
|
128
165
|
this.initializeNodeStats();
|
|
129
166
|
this.loadBlacklistState();
|
|
167
|
+
this.loadChurnHistory();
|
|
130
168
|
// Blacklist warning dedup: track last warn time per node+error to suppress repeats.
|
|
131
169
|
this._lastBlacklistWarnMs = new Map();
|
|
132
170
|
// Control flags
|
|
@@ -155,10 +193,8 @@ class NodeManager {
|
|
|
155
193
|
lastErrorMessage: null,
|
|
156
194
|
chainId: null,
|
|
157
195
|
blacklistedAt: null,
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
healthCheckSuccessStreak: 0,
|
|
161
|
-
lastFailureSource: null
|
|
196
|
+
lastFailureAtMs: null,
|
|
197
|
+
dropTimes: []
|
|
162
198
|
});
|
|
163
199
|
}
|
|
164
200
|
}
|
|
@@ -182,6 +218,7 @@ class NodeManager {
|
|
|
182
218
|
stats.failureCount = typeof entry.failureCount === 'number' ? entry.failureCount : this.config.healthCheck.blacklistThreshold;
|
|
183
219
|
stats.blacklistedAt = typeof entry.blacklistedAt === 'number' ? entry.blacklistedAt : Date.now();
|
|
184
220
|
stats.lastErrorMessage = typeof entry.lastErrorMessage === 'string' ? entry.lastErrorMessage : null;
|
|
221
|
+
stats.lastFailureAtMs = typeof entry.lastFailureAtMs === 'number' ? entry.lastFailureAtMs : null;
|
|
185
222
|
// Seed the shared ledger so counting continues across restarts
|
|
186
223
|
// (a persisted blacklist must still count toward the threshold
|
|
187
224
|
// until the cooldown expires or a health check re-admits the
|
|
@@ -213,7 +250,8 @@ class NodeManager {
|
|
|
213
250
|
status: 'blacklisted',
|
|
214
251
|
blacklistedAt: stats.blacklistedAt,
|
|
215
252
|
failureCount: stats.failureCount,
|
|
216
|
-
lastErrorMessage: stats.lastErrorMessage
|
|
253
|
+
lastErrorMessage: stats.lastErrorMessage,
|
|
254
|
+
lastFailureAtMs: stats.lastFailureAtMs
|
|
217
255
|
};
|
|
218
256
|
}
|
|
219
257
|
}
|
|
@@ -226,6 +264,44 @@ class NodeManager {
|
|
|
226
264
|
this.logger.warn(`Failed to save blacklist state: ${getErrorMessage(err)}`);
|
|
227
265
|
}
|
|
228
266
|
}
|
|
267
|
+
/**
|
|
268
|
+
* Seed the in-memory drop history from the shared health cache.
|
|
269
|
+
*
|
|
270
|
+
* Churn has to survive process restarts: a bot that is restarted every few
|
|
271
|
+
* hours would otherwise start every node at zero drops and re-select the
|
|
272
|
+
* flapping endpoint immediately. The cache is written by whichever process
|
|
273
|
+
* last ran a health sweep, so treat it as advisory (any parse/age problem
|
|
274
|
+
* just means "no history yet").
|
|
275
|
+
* @private
|
|
276
|
+
*/
|
|
277
|
+
loadChurnHistory() {
|
|
278
|
+
try {
|
|
279
|
+
const cache = readHealthCache({
|
|
280
|
+
healthCacheFile: this.healthCacheFile,
|
|
281
|
+
// Keep history that is newer than the flap window; a 4h probe
|
|
282
|
+
// staleness limit would throw away a valid 24h drop history.
|
|
283
|
+
maxAgeMs: NODE_MANAGEMENT.FLAP_WINDOW_MS,
|
|
284
|
+
});
|
|
285
|
+
if (!cache || !Array.isArray(cache.nodes))
|
|
286
|
+
return;
|
|
287
|
+
const now = Date.now();
|
|
288
|
+
for (const entry of cache.nodes) {
|
|
289
|
+
const stats = entry?.url ? this.nodeStats.get(entry.url) : undefined;
|
|
290
|
+
if (!stats || !Array.isArray(entry.dropTimes) || entry.dropTimes.length === 0)
|
|
291
|
+
continue;
|
|
292
|
+
const flapOptions = this._flapOptions(now);
|
|
293
|
+
stats.dropTimes = pruneDropTimes(entry.dropTimes, now, flapOptions.windowMs);
|
|
294
|
+
if (stats.dropTimes.length > 0) {
|
|
295
|
+
const counters = dropCounters(stats.dropTimes, flapOptions);
|
|
296
|
+
this.logger.debug(`Loaded churn history for ${entry.url.substring(0, 40)}... ` +
|
|
297
|
+
`(${counters.drops24h} drop(s)/24h, penalty ~${counters.penaltyMs}ms)`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
catch (err) {
|
|
302
|
+
this.logger.debug(`Churn history unavailable: ${getErrorMessage(err)}`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
229
305
|
/**
|
|
230
306
|
* Persist the latest healthy/slow node ordering for lightweight consumers
|
|
231
307
|
* such as the credential daemon.
|
|
@@ -233,7 +309,12 @@ class NodeManager {
|
|
|
233
309
|
*/
|
|
234
310
|
saveHealthCache() {
|
|
235
311
|
try {
|
|
236
|
-
writeHealthCache(this.nodeStats.values(), {
|
|
312
|
+
writeHealthCache(this.nodeStats.values(), {
|
|
313
|
+
healthCacheFile: this.healthCacheFile,
|
|
314
|
+
// Pass this manager's tuning so the persisted drops/h counters
|
|
315
|
+
// and ordering match what the bot itself would compute.
|
|
316
|
+
flap: this._flapOptions(),
|
|
317
|
+
});
|
|
237
318
|
}
|
|
238
319
|
catch (err) {
|
|
239
320
|
this.logger.warn(`Failed to save node health cache: ${getErrorMessage(err)}`);
|
|
@@ -302,6 +383,10 @@ class NodeManager {
|
|
|
302
383
|
stats.status = 'unchecked';
|
|
303
384
|
stats.failureCount = 0;
|
|
304
385
|
stats.blacklistedAt = null;
|
|
386
|
+
stats.lastFailureAtMs = null;
|
|
387
|
+
// Churn history is intentionally kept: a node coming off
|
|
388
|
+
// blacklist still has to prove stability, and the flap
|
|
389
|
+
// penalty is what keeps it behind a node that never drops.
|
|
305
390
|
this.failureLedger.reset(nodeUrl);
|
|
306
391
|
this.saveBlacklistState();
|
|
307
392
|
this.logger.info(`${nodeUrl.substring(0, 40)}... blacklist cooldown expired, re-enabling`);
|
|
@@ -375,41 +460,66 @@ class NodeManager {
|
|
|
375
460
|
}
|
|
376
461
|
}
|
|
377
462
|
/**
|
|
378
|
-
*
|
|
379
|
-
*
|
|
463
|
+
* Remaining (decayed) strikes for a node at `nowMs`.
|
|
464
|
+
*
|
|
465
|
+
* A strike is worth 1.0 when recorded and falls linearly to 0 across
|
|
466
|
+
* `strikeDecayMs`. Linear decay (not exponential) keeps the threshold
|
|
467
|
+
* meaningful: three strikes on the same day still blacklist, while a node
|
|
468
|
+
* that failed once three days ago starts from 0 again. The remainder is
|
|
469
|
+
* rounded *up*, so a fresh single strike is not silently discounted to 0
|
|
470
|
+
* within minutes by the rounding step.
|
|
471
|
+
*/
|
|
472
|
+
_decayedStrikeCount(stats, nowMs = Date.now()) {
|
|
473
|
+
if (stats.failureCount <= 0)
|
|
474
|
+
return 0;
|
|
475
|
+
if (!stats.lastFailureAtMs)
|
|
476
|
+
return stats.failureCount;
|
|
477
|
+
const decayMs = this.config.healthCheck.strikeDecayMs;
|
|
478
|
+
const age = Math.max(0, nowMs - stats.lastFailureAtMs);
|
|
479
|
+
if (age >= decayMs)
|
|
480
|
+
return 0;
|
|
481
|
+
const remainingFactor = 1 - age / decayMs;
|
|
482
|
+
return Math.max(0, Math.ceil(stats.failureCount * remainingFactor));
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* Record a successful health-probe result: set status/latency and let the
|
|
486
|
+
* strike count decay.
|
|
380
487
|
*
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
*
|
|
488
|
+
* This deliberately no longer *resets* strikes: the old
|
|
489
|
+
* liveFailureSuccessStreak rule cleared a live-transport strike after two
|
|
490
|
+
* successful probes, and with a 4h probe interval that forgave a
|
|
491
|
+
* once-a-day flapper twice a day — every failure logged "attempt 1/3" and
|
|
492
|
+
* the node was never blacklisted. Strikes now age out on their own
|
|
493
|
+
* (`strikeDecayMs`), so probe success can re-admit a node's *status*
|
|
494
|
+
* without erasing the evidence that it drops connections.
|
|
387
495
|
*
|
|
388
|
-
* @returns {{ recovered: boolean, failureCount: number }}
|
|
496
|
+
* @returns {{ recovered: boolean, failureCount: number }} whether every
|
|
497
|
+
* strike has aged out, and the remaining count.
|
|
389
498
|
*/
|
|
390
499
|
_recordHealthCheckSuccess(nodeUrl, status, latencyMs, chainId) {
|
|
391
500
|
const stats = this.nodeStats.get(nodeUrl);
|
|
392
501
|
if (!stats)
|
|
393
502
|
return { recovered: false, failureCount: 0 };
|
|
394
|
-
const priorSource = stats.lastFailureSource;
|
|
395
|
-
const liveFailure = !!priorSource && priorSource !== 'health-check';
|
|
396
|
-
stats.healthCheckSuccessStreak = (stats.healthCheckSuccessStreak || 0) + 1;
|
|
397
|
-
const successStreak = this.config.healthCheck.liveFailureSuccessStreak;
|
|
398
|
-
const recovered = !liveFailure || stats.healthCheckSuccessStreak >= successStreak;
|
|
399
503
|
stats.status = status;
|
|
400
504
|
stats.latencyMs = latencyMs;
|
|
401
505
|
stats.lastCheckTime = nowIso();
|
|
402
506
|
stats.lastErrorMessage = null;
|
|
403
507
|
stats.chainId = chainId;
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
508
|
+
const remaining = this._decayedStrikeCount(stats);
|
|
509
|
+
const recovered = remaining <= 0;
|
|
510
|
+
if (stats.failureCount > 0) {
|
|
511
|
+
if (remaining <= 0) {
|
|
512
|
+
stats.failureCount = 0;
|
|
513
|
+
stats.lastFailureAtMs = null;
|
|
514
|
+
this.failureLedger.reset(nodeUrl);
|
|
515
|
+
}
|
|
516
|
+
else {
|
|
517
|
+
stats.failureCount = remaining;
|
|
518
|
+
// Keep the ledger and the visible count in step, otherwise the
|
|
519
|
+
// next failure would blacklist on stale evidence.
|
|
520
|
+
this.failureLedger.seed(nodeUrl, { failureCount: remaining, blacklistedUntil: 0 });
|
|
521
|
+
this.logger.debug(`${nodeUrl.substring(0, 40)}... ${status} but retaining ${remaining} decaying strike(s)`);
|
|
522
|
+
}
|
|
413
523
|
}
|
|
414
524
|
return { recovered, failureCount: stats.failureCount };
|
|
415
525
|
}
|
|
@@ -567,13 +677,40 @@ class NodeManager {
|
|
|
567
677
|
});
|
|
568
678
|
}
|
|
569
679
|
/**
|
|
570
|
-
*
|
|
680
|
+
* Flap-scoring options built from this manager's configuration.
|
|
681
|
+
* Single place that maps config -> the pure ranking maths, so the ranking,
|
|
682
|
+
* the logged counters and the persisted health cache always agree.
|
|
683
|
+
* @private
|
|
684
|
+
*/
|
|
685
|
+
_flapOptions(nowMs = Date.now()) {
|
|
686
|
+
return {
|
|
687
|
+
nowMs,
|
|
688
|
+
windowMs: NODE_MANAGEMENT.FLAP_WINDOW_MS,
|
|
689
|
+
halfLifeMs: this.config.healthCheck.flapHalfLifeMs,
|
|
690
|
+
penaltyMsPerDrop: this.config.healthCheck.flapPenaltyMsPerDrop,
|
|
691
|
+
maxPenaltyMs: this.config.healthCheck.flapPenaltyMaxMs,
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
/**
|
|
695
|
+
* Get list of healthy nodes, best first.
|
|
696
|
+
*
|
|
697
|
+
* Ranking is `measured latency + decayed connection-churn penalty`, so a
|
|
698
|
+
* 50ms node that drops its socket daily loses to a stable 350ms node. See
|
|
699
|
+
* modules/node_flap_score.ts for the penalty maths.
|
|
571
700
|
* @returns {string[]} Array of node URLs
|
|
572
701
|
*/
|
|
573
702
|
getHealthyNodes() {
|
|
574
703
|
const preferredNode = this.config.selection.preferredNode;
|
|
575
|
-
const
|
|
576
|
-
|
|
704
|
+
const flapOptions = this._flapOptions();
|
|
705
|
+
const healthyStats = Array.from(this.nodeStats.values())
|
|
706
|
+
.filter((stat) => stat.status === 'healthy' || stat.status === 'slow');
|
|
707
|
+
// Precompute one rank per node: doing it inside the comparator would
|
|
708
|
+
// prune and re-sort every node's drop history O(n log n) times.
|
|
709
|
+
const ranks = new Map();
|
|
710
|
+
for (const stat of healthyStats) {
|
|
711
|
+
ranks.set(stat, effectiveLatencyMs(stat.latencyMs, stat.dropTimes, flapOptions));
|
|
712
|
+
}
|
|
713
|
+
const healthy = healthyStats
|
|
577
714
|
.sort((a, b) => {
|
|
578
715
|
if (preferredNode) {
|
|
579
716
|
const aPreferred = a.url === preferredNode && a.status === 'healthy';
|
|
@@ -581,13 +718,36 @@ class NodeManager {
|
|
|
581
718
|
if (aPreferred !== bPreferred)
|
|
582
719
|
return aPreferred ? -1 : 1;
|
|
583
720
|
}
|
|
584
|
-
|
|
721
|
+
if (a.status !== b.status)
|
|
722
|
+
return a.status === 'healthy' ? -1 : 1;
|
|
723
|
+
// Compare without subtracting: two unknown latencies are both
|
|
724
|
+
// Infinity, and Infinity - Infinity is NaN (an unstable sort
|
|
725
|
+
// comparator).
|
|
726
|
+
const aRank = ranks.get(a) ?? Infinity;
|
|
727
|
+
const bRank = ranks.get(b) ?? Infinity;
|
|
728
|
+
if (aRank === bRank)
|
|
729
|
+
return 0;
|
|
730
|
+
return aRank < bRank ? -1 : 1;
|
|
585
731
|
})
|
|
586
732
|
.map((stat) => stat.url);
|
|
587
733
|
return healthy;
|
|
588
734
|
}
|
|
589
735
|
/**
|
|
590
|
-
*
|
|
736
|
+
* Connection-churn counters for one node (drops/h, decayed score, penalty).
|
|
737
|
+
* Used for logging and for the shared health cache.
|
|
738
|
+
* @param {string} nodeUrl - Node URL
|
|
739
|
+
* @returns {Object|null} Churn counters or null for an unknown node
|
|
740
|
+
*/
|
|
741
|
+
getNodeChurn(nodeUrl, nowMs = Date.now()) {
|
|
742
|
+
const stats = this.nodeStats.get(nodeUrl);
|
|
743
|
+
if (!stats)
|
|
744
|
+
return null;
|
|
745
|
+
const flapOptions = this._flapOptions(nowMs);
|
|
746
|
+
const dropTimes = pruneDropTimes(stats.dropTimes, nowMs, flapOptions.windowMs);
|
|
747
|
+
return { dropTimes, ...dropCounters(dropTimes, flapOptions) };
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* Get the single best-ranked healthy node (latency + churn penalty)
|
|
591
751
|
* @returns {string|null} Best node URL or null if none available
|
|
592
752
|
*/
|
|
593
753
|
getBestNode() {
|
|
@@ -621,41 +781,55 @@ class NodeManager {
|
|
|
621
781
|
* failure ledger — the same rules the credential daemon applies.
|
|
622
782
|
* @param {string} nodeUrl - Node URL that failed
|
|
623
783
|
* @param {string} [errorMessage] - Optional error description
|
|
624
|
-
* @param {string} [source] - Failure source ('health-check'
|
|
784
|
+
* @param {string} [source] - Failure source ('health-check', 'broadcast',
|
|
785
|
+
* 'connection', 'keep-alive'); controls log level and churn accounting
|
|
786
|
+
* @param {number} [nowMs] - Timestamp for the strike/drop (defaults to now;
|
|
787
|
+
* injectable so tests do not need to sleep past the report cooldown)
|
|
625
788
|
*/
|
|
626
|
-
reportNodeFailure(nodeUrl, errorMessage, source) {
|
|
789
|
+
reportNodeFailure(nodeUrl, errorMessage, source, nowMs = Date.now()) {
|
|
627
790
|
const stats = this.nodeStats.get(nodeUrl);
|
|
628
791
|
if (!stats) {
|
|
629
792
|
this.logger.warn(`Node ${nodeUrl} not in configured list, cannot report failure`);
|
|
630
793
|
return;
|
|
631
794
|
}
|
|
632
795
|
const isHealthCheck = source === 'health-check';
|
|
633
|
-
const result = this.failureLedger.recordFailure(nodeUrl);
|
|
796
|
+
const result = this.failureLedger.recordFailure(nodeUrl, nowMs);
|
|
634
797
|
stats.failureCount = result.failureCount;
|
|
635
798
|
stats.lastCheckTime = nowIso();
|
|
636
799
|
if (errorMessage)
|
|
637
800
|
stats.lastErrorMessage = errorMessage;
|
|
638
|
-
|
|
639
|
-
// strike
|
|
640
|
-
//
|
|
641
|
-
|
|
642
|
-
|
|
801
|
+
const counted = result.outcome === 'counted' || result.outcome === 'blacklisted';
|
|
802
|
+
// A counted strike refreshes the decay clock. Recording it even for
|
|
803
|
+
// probe failures keeps `failureCount` and its age consistent.
|
|
804
|
+
if (counted) {
|
|
805
|
+
stats.lastFailureAtMs = nowMs;
|
|
806
|
+
}
|
|
807
|
+
// Only live-operation failures feed the churn score used by selection
|
|
808
|
+
// (see CHURN_SOURCES). Rate-limited reports restate the same event, so
|
|
809
|
+
// they are excluded along with blacklisted-node repeats.
|
|
810
|
+
if (counted && source && CHURN_SOURCES.has(source)) {
|
|
811
|
+
stats.dropTimes = recordDrop(stats.dropTimes, nowMs, this._flapOptions(nowMs).windowMs);
|
|
812
|
+
}
|
|
643
813
|
if (result.outcome === 'rate-limited' || result.outcome === 'skipped-blacklisted') {
|
|
644
814
|
return;
|
|
645
815
|
}
|
|
646
816
|
if (result.outcome === 'blacklisted') {
|
|
647
817
|
stats.status = 'blacklisted';
|
|
648
|
-
stats.blacklistedAt =
|
|
818
|
+
stats.blacklistedAt = nowMs;
|
|
649
819
|
this.saveBlacklistState();
|
|
650
820
|
this.saveHealthCache();
|
|
651
821
|
if (this._shouldLogBlacklistWarning(nodeUrl, errorMessage || '')) {
|
|
652
|
-
|
|
822
|
+
const churn = this.getNodeChurn(nodeUrl);
|
|
823
|
+
this.logger.warn(`✗ ${nodeUrl.substring(0, 40)}... BLACKLISTED after ${result.failureCount} failures (${errorMessage || 'unknown'}) ` +
|
|
824
|
+
`[churn ${churn ? `${churn.drops24h} drop(s)/24h, +${churn.penaltyMs}ms` : 'n/a'}]`);
|
|
653
825
|
}
|
|
654
826
|
}
|
|
655
827
|
else {
|
|
656
828
|
stats.status = 'failed';
|
|
657
829
|
const log = isHealthCheck ? this.logger.debug.bind(this.logger) : this.logger.warn.bind(this.logger);
|
|
658
|
-
|
|
830
|
+
const churn = this.getNodeChurn(nodeUrl);
|
|
831
|
+
log(`⚠ ${nodeUrl.substring(0, 40)}... FAILED attempt ${result.failureCount}/${result.threshold} (${errorMessage || 'unknown'})` +
|
|
832
|
+
(churn && churn.drops24h > 0 ? ` [churn ${churn.drops24h} drop(s)/24h, +${churn.penaltyMs}ms]` : ''));
|
|
659
833
|
}
|
|
660
834
|
}
|
|
661
835
|
/**
|
|
@@ -668,6 +842,7 @@ class NodeManager {
|
|
|
668
842
|
stats.status = 'blacklisted';
|
|
669
843
|
stats.failureCount = this.config.healthCheck.blacklistThreshold;
|
|
670
844
|
stats.blacklistedAt = Date.now();
|
|
845
|
+
stats.lastFailureAtMs = stats.blacklistedAt;
|
|
671
846
|
this.failureLedger.seed(nodeUrl, {
|
|
672
847
|
failureCount: stats.failureCount,
|
|
673
848
|
blacklistedUntil: stats.blacklistedAt + this.BLACKLIST_COOLDOWN_MS,
|
|
@@ -689,8 +864,10 @@ class NodeManager {
|
|
|
689
864
|
stats.latencyMs = null;
|
|
690
865
|
stats.lastErrorMessage = null;
|
|
691
866
|
stats.blacklistedAt = null;
|
|
692
|
-
stats.
|
|
693
|
-
|
|
867
|
+
stats.lastFailureAtMs = null;
|
|
868
|
+
// An explicit reset is the operator saying "give this node a clean
|
|
869
|
+
// slate": churn history goes with the strikes.
|
|
870
|
+
stats.dropTimes = [];
|
|
694
871
|
this.failureLedger.reset(nodeUrl);
|
|
695
872
|
this._clearBlacklistWarningCooldown(nodeUrl);
|
|
696
873
|
this.saveBlacklistState();
|
|
@@ -709,8 +886,8 @@ class NodeManager {
|
|
|
709
886
|
stats.lastErrorMessage = null;
|
|
710
887
|
stats.chainId = null;
|
|
711
888
|
stats.blacklistedAt = null;
|
|
712
|
-
stats.
|
|
713
|
-
stats.
|
|
889
|
+
stats.lastFailureAtMs = null;
|
|
890
|
+
stats.dropTimes = [];
|
|
714
891
|
}
|
|
715
892
|
this.failureLedger.clear();
|
|
716
893
|
this._clearBlacklistWarningCooldown();
|
|
@@ -723,14 +900,21 @@ class NodeManager {
|
|
|
723
900
|
* @returns {Array<Object>} Array of node stats
|
|
724
901
|
*/
|
|
725
902
|
getStats() {
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
903
|
+
const nowMs = Date.now();
|
|
904
|
+
return Array.from(this.nodeStats.values()).map((stat) => {
|
|
905
|
+
const churn = this.getNodeChurn(stat.url, nowMs);
|
|
906
|
+
return {
|
|
907
|
+
url: stat.url,
|
|
908
|
+
status: stat.status,
|
|
909
|
+
latencyMs: stat.latencyMs,
|
|
910
|
+
failureCount: stat.failureCount,
|
|
911
|
+
lastCheckTime: stat.lastCheckTime,
|
|
912
|
+
lastErrorMessage: stat.lastErrorMessage,
|
|
913
|
+
drops24h: churn ? churn.drops24h : 0,
|
|
914
|
+
dropsPerHour: churn ? churn.dropsPerHour : 0,
|
|
915
|
+
flapPenaltyMs: churn ? churn.penaltyMs : 0
|
|
916
|
+
};
|
|
917
|
+
});
|
|
734
918
|
}
|
|
735
919
|
/**
|
|
736
920
|
* Get summary statistics
|
|
@@ -755,11 +939,17 @@ class NodeManager {
|
|
|
755
939
|
const avgLatency = statsWithLatency.length > 0
|
|
756
940
|
? Math.round(statsWithLatency.reduce((sum, s) => sum + s.latencyMs, 0) / statsWithLatency.length)
|
|
757
941
|
: null;
|
|
942
|
+
const nowMs = Date.now();
|
|
943
|
+
const totalDrops24h = stats.reduce((sum, s) => {
|
|
944
|
+
const churn = this.getNodeChurn(s.url, nowMs);
|
|
945
|
+
return sum + (churn ? churn.drops24h : 0);
|
|
946
|
+
}, 0);
|
|
758
947
|
return {
|
|
759
948
|
monitoring: this.monitoringActive,
|
|
760
949
|
counts,
|
|
761
950
|
bestNode,
|
|
762
|
-
avgLatency
|
|
951
|
+
avgLatency,
|
|
952
|
+
totalDrops24h
|
|
763
953
|
};
|
|
764
954
|
}
|
|
765
955
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* Exports a single Accountant class that manages all fund accounting operations.
|
|
7
7
|
*
|
|
8
8
|
* ===============================================================================
|
|
9
|
-
* TABLE OF CONTENTS - Accountant Class (
|
|
9
|
+
* TABLE OF CONTENTS - Accountant Class (27 methods)
|
|
10
10
|
* ===============================================================================
|
|
11
11
|
*
|
|
12
12
|
* CORE INITIALIZATION & RECALCULATION (2 methods)
|
|
@@ -56,6 +56,7 @@
|
|
|
56
56
|
* 25. _recalibrateTrackedFundsFromChain(mgr, openOrders) - Recalibrate tracked funds
|
|
57
57
|
* from a fresh chain snapshot
|
|
58
58
|
* 26. _tryTrustChainFreeHeal(mgr, driftValidation) - Attempt a trust-chain-free heal
|
|
59
|
+
* 27. _getBtsFeeSchedule() - Resolve the BTS fee schedule (create/update/cancel + maker refund %)
|
|
59
60
|
*
|
|
60
61
|
* ===============================================================================
|
|
61
62
|
* FUND STRUCTURE (managed by Accountant)
|
|
@@ -131,9 +132,13 @@ declare class Accountant {
|
|
|
131
132
|
constructor(manager: OrderManagerLike);
|
|
132
133
|
_logThrottled(key: string, message: string, level?: string, intervalMs?: number): boolean;
|
|
133
134
|
/**
|
|
134
|
-
* Apply
|
|
135
|
-
*
|
|
136
|
-
*
|
|
135
|
+
* Apply balance adjustments ALL-OR-NOTHING.
|
|
136
|
+
*
|
|
137
|
+
* A throw from the middle of the list used to leave the earlier deltas
|
|
138
|
+
* applied. Since P2 releases the dedup claim on failure (so the fill can be
|
|
139
|
+
* retried), a partial application would then be applied twice on the retry.
|
|
140
|
+
* The credit is now atomic: an applied prefix is reversed before the error
|
|
141
|
+
* propagates, so "not credited" is true whenever the claim was released.
|
|
137
142
|
*/
|
|
138
143
|
_applyBalanceAdjustments(balanceAdjustments: Array<{
|
|
139
144
|
orderType: string | null | undefined;
|