dexbot 1.7.0 → 1.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +3 -3
  3. package/analysis/README.md +33 -13
  4. package/claw/package.json +1 -1
  5. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  6. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  7. package/dist/analysis/account_resolver.js +5 -6
  8. package/dist/analysis/analyze_liquidity.js +15 -4
  9. package/dist/analysis/bot_key_utils.d.ts +44 -1
  10. package/dist/analysis/bot_key_utils.js +76 -7
  11. package/dist/analysis/fills_cache.js +2 -1
  12. package/dist/analysis/grid_correction_check.d.ts +1 -19
  13. package/dist/analysis/grid_correction_check.js +12 -30
  14. package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
  15. package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
  16. package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
  17. package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
  18. package/dist/analysis/liquidity/liquidity_view.js +11 -0
  19. package/dist/analysis/resolve_bot_accounts.js +7 -6
  20. package/dist/analysis/trade_profitability.js +2 -1
  21. package/dist/credential-daemon.js +13 -2
  22. package/dist/dexbot.js +4 -4
  23. package/dist/modules/account_bots.d.ts +12 -1
  24. package/dist/modules/account_bots.js +7 -5
  25. package/dist/modules/account_orders.d.ts +12 -0
  26. package/dist/modules/account_orders.js +30 -0
  27. package/dist/modules/bitshares-native/transport.js +24 -6
  28. package/dist/modules/bitshares_client.d.ts +24 -6
  29. package/dist/modules/bitshares_client.js +44 -7
  30. package/dist/modules/bot_settings.d.ts +2 -0
  31. package/dist/modules/constants.d.ts +28 -3
  32. package/dist/modules/constants.js +79 -10
  33. package/dist/modules/dexbot_class.d.ts +23 -6
  34. package/dist/modules/dexbot_class.js +44 -0
  35. package/dist/modules/dexbot_cow_runtime.d.ts +128 -8
  36. package/dist/modules/dexbot_cow_runtime.js +688 -188
  37. package/dist/modules/dexbot_fill_runtime.js +124 -35
  38. package/dist/modules/dexbot_maintenance_runtime.d.ts +1 -1
  39. package/dist/modules/dexbot_maintenance_runtime.js +259 -79
  40. package/dist/modules/dexbot_startup_runtime.js +32 -20
  41. package/dist/modules/dexbot_state_recovery.d.ts +33 -1
  42. package/dist/modules/dexbot_state_recovery.js +167 -5
  43. package/dist/modules/node_connect_policy.d.ts +8 -1
  44. package/dist/modules/node_connect_policy.js +18 -2
  45. package/dist/modules/node_flap_score.d.ts +87 -0
  46. package/dist/modules/node_flap_score.js +124 -0
  47. package/dist/modules/node_health_cache.d.ts +40 -3
  48. package/dist/modules/node_health_cache.js +66 -20
  49. package/dist/modules/node_manager.d.ts +101 -19
  50. package/dist/modules/node_manager.js +255 -65
  51. package/dist/modules/order/accounting.d.ts +9 -4
  52. package/dist/modules/order/accounting.js +131 -19
  53. package/dist/modules/order/async_lock.d.ts +7 -1
  54. package/dist/modules/order/async_lock.js +7 -1
  55. package/dist/modules/order/export.js +25 -18
  56. package/dist/modules/order/fill_ledger.d.ts +99 -0
  57. package/dist/modules/order/fill_ledger.js +103 -0
  58. package/dist/modules/order/format.d.ts +24 -10
  59. package/dist/modules/order/format.js +30 -10
  60. package/dist/modules/order/grid.d.ts +75 -18
  61. package/dist/modules/order/grid.js +289 -72
  62. package/dist/modules/order/grid_reconcile.d.ts +5 -1
  63. package/dist/modules/order/grid_reconcile.js +333 -210
  64. package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
  65. package/dist/modules/order/grid_reconcile_internal.js +223 -91
  66. package/dist/modules/order/index.d.ts +18 -5
  67. package/dist/modules/order/index.js +5 -5
  68. package/dist/modules/order/lifecycle.d.ts +353 -0
  69. package/dist/modules/order/lifecycle.js +1109 -0
  70. package/dist/modules/order/manager.d.ts +277 -20
  71. package/dist/modules/order/manager.js +680 -64
  72. package/dist/modules/order/placement_lock.d.ts +102 -0
  73. package/dist/modules/order/placement_lock.js +222 -0
  74. package/dist/modules/order/strategy.js +13 -1
  75. package/dist/modules/order/sync_engine.d.ts +74 -114
  76. package/dist/modules/order/sync_engine.js +254 -198
  77. package/dist/modules/order/utils/math.d.ts +29 -4
  78. package/dist/modules/order/utils/math.js +61 -4
  79. package/dist/modules/order/utils/order.d.ts +106 -37
  80. package/dist/modules/order/utils/order.js +256 -81
  81. package/dist/modules/order/utils/system.d.ts +99 -6
  82. package/dist/modules/order/utils/system.js +249 -35
  83. package/dist/modules/order/utils/validate.d.ts +7 -1
  84. package/dist/modules/order/utils/validate.js +48 -2
  85. package/dist/modules/outcome.d.ts +54 -0
  86. package/dist/modules/outcome.js +51 -0
  87. package/dist/modules/settings_merge.d.ts +4 -0
  88. package/dist/modules/settings_merge.js +9 -0
  89. package/dist/modules/timing_policy.d.ts +82 -0
  90. package/dist/modules/timing_policy.js +244 -0
  91. package/dist/modules/types.d.ts +149 -22
  92. package/dist/scripts/analyze-credit.js +54 -17
  93. package/dist/scripts/analyze-orders.d.ts +4 -2
  94. package/dist/scripts/analyze-orders.js +113 -34
  95. package/dist/scripts/chart_targets.js +11 -15
  96. package/dist/scripts/test-credit-renewal.js +12 -5
  97. package/docs/COW_INVARIANTS.md +71 -8
  98. package/docs/DEXBOT_COMPARISON.md +6 -6
  99. package/docs/EVOLUTION.md +10 -8
  100. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  101. package/docs/GRID_PRICE_INVARIANT.md +1 -1
  102. package/docs/GRID_RECONCILE.md +18 -13
  103. package/docs/LOGGING.md +25 -1
  104. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +183 -26
  105. package/docs/README.md +23 -2
  106. package/docs/WORKFLOW.md +9 -3
  107. package/docs/architecture.md +68 -13
  108. package/docs/developer_guide.md +25 -8
  109. package/modules/README.md +6 -0
  110. package/package.json +3 -3
  111. package/scripts/README.md +16 -0
  112. package/scripts/pdev.sh +14 -1
  113. package/scripts/pmain.sh +23 -8
  114. package/scripts/ptest.sh +7 -0
  115. package/tests/README.md +30 -1
@@ -15,7 +15,9 @@
15
15
  * - Automatic blacklisting of failed nodes
16
16
  *
17
17
  * 2. Intelligent Node Selection
18
- * - Selects lowest-latency healthy node
18
+ * - Ranks nodes by measured latency plus a time-decayed connection-churn
19
+ * penalty (see modules/node_flap_score.ts), so a fast node that keeps
20
+ * dropping its socket loses to a stable slightly-slower one
19
21
  * - Falls back to slow nodes if no healthy ones available
20
22
  * - Respects user-preferred node if healthy
21
23
  *
@@ -44,19 +46,49 @@ const require = createRequire(import.meta.url);
44
46
  import Logger from './order/logger.js';
45
47
  import { NODE_MANAGEMENT } from './constants.js';
46
48
  import { PATHS, getNodeBlacklistFile } from './paths.js';
47
- import { compareNodeHealth } from './node_health_cache.js';
48
49
  import { writeJsonFileAtomic } from './bots_file_lock.js';
49
50
  import { getStorage } from './storage/index.js';
50
51
  const storage = getStorage();
51
52
  const { readJSON } = storage;
52
53
  import { createFailureLedger } from './node_failure_ledger.js';
53
54
  const _WebSocket = globalThis.WebSocket;
54
- import { resolveHealthCacheFile, writeHealthCache, } from './node_health_cache.js';
55
+ import { readHealthCache, resolveHealthCacheFile, writeHealthCache, } from './node_health_cache.js';
56
+ import { dropCounters, effectiveLatencyMs, pruneDropTimes, recordDrop, } from './node_flap_score.js';
55
57
  import { getErrorMessage, getErrorCode } from './utils/errors.js';
56
58
  import { nowIso } from './order/utils/system.js';
59
+ /**
60
+ * Failure sources that count as a live-operation reliability event for churn
61
+ * scoring/selection.
62
+ *
63
+ * Included:
64
+ * - `connection` an unexpected socket close (code 1006/1001 etc.)
65
+ * - `keep-alive` the socket stayed open but stopped answering
66
+ * - `broadcast` a signed batch was accepted but never confirmed in time
67
+ * (BROADCAST_DEADLINE) — the socket is up, the node is not
68
+ * serving us, which is exactly the instability selection
69
+ * should penalize
70
+ *
71
+ * Excluded:
72
+ * - `health-check` probe-side noise; it already moves the node out of the
73
+ * healthy set, and probes may fail for reasons that never
74
+ * affect a live session
75
+ * - `forced-reconnect` a client-side recovery attempt, not evidence that the
76
+ * node is bad (same rule as shouldCountNodeStrike in
77
+ * bitshares_client)
78
+ */
79
+ const CHURN_SOURCES = new Set(['connection', 'keep-alive', 'broadcast']);
80
+ /** Positive finite number or the fallback (config may carry 0/NaN/strings). */
81
+ function positiveOr(value, fallback) {
82
+ const n = Number(value);
83
+ return Number.isFinite(n) && n > 0 ? n : fallback;
84
+ }
57
85
  /**
58
86
  * NodeManager - Manages health checking and selection of BitShares nodes
59
87
  *
88
+ * Selection ranks healthy nodes on `latency + decayed churn penalty`, keeps a
89
+ * time-decaying strike count for blacklisting, and exposes per-node churn
90
+ * counters (drops/h) for logging and the shared health cache.
91
+ *
60
92
  * @class
61
93
  * @param {Object} config - Configuration object
62
94
  * @param {string[]} config.list - List of node URLs to monitor (e.g., wss://dex.iobanker.com/ws)
@@ -65,8 +97,12 @@ import { nowIso } from './order/utils/system.js';
65
97
  * @param {number} [config.healthCheck.timeoutMs=5000] - Timeout for each health check
66
98
  * @param {number} [config.healthCheck.maxPingMs=3000] - Max acceptable latency
67
99
  * @param {number} [config.healthCheck.blacklistThreshold=3] - Failures before blacklist
100
+ * @param {number} [config.healthCheck.strikeDecayMs] - Lifetime of a strike without a new failure
101
+ * @param {number} [config.healthCheck.flapHalfLifeMs] - Half-life of a connection-drop penalty
102
+ * @param {number} [config.healthCheck.flapPenaltyMsPerDrop] - Effective-latency cost per fresh drop
103
+ * @param {number} [config.healthCheck.flapPenaltyMaxMs] - Cap on the decayed churn penalty
68
104
  * @param {Object} [config.selection] - Node selection settings
69
- * @param {string} [config.selection.strategy="latency"] - Selection strategy
105
+ * @param {string} [config.selection.strategy="latency"] - Selection strategy (latency + churn penalty)
70
106
  * @param {string} [config.selection.preferredNode] - Optional preferred node (if healthy)
71
107
  */
72
108
  class NodeManager {
@@ -98,9 +134,10 @@ class NodeManager {
98
134
  timeoutMs: config.healthCheck?.timeoutMs ?? NODE_MANAGEMENT.HEALTH_CHECK_TIMEOUT_MS,
99
135
  maxPingMs: config.healthCheck?.maxPingMs ?? NODE_MANAGEMENT.MAX_PING_MS,
100
136
  blacklistThreshold: config.healthCheck?.blacklistThreshold ?? NODE_MANAGEMENT.BLACKLIST_THRESHOLD,
101
- liveFailureSuccessStreak: Number(config.healthCheck?.liveFailureSuccessStreak) > 0
102
- ? Number(config.healthCheck?.liveFailureSuccessStreak)
103
- : NODE_MANAGEMENT.LIVE_FAILURE_HEALTH_SUCCESS_STREAK,
137
+ strikeDecayMs: positiveOr(config.healthCheck?.strikeDecayMs, NODE_MANAGEMENT.FAILURE_STRIKE_DECAY_MS),
138
+ flapHalfLifeMs: positiveOr(config.healthCheck?.flapHalfLifeMs, NODE_MANAGEMENT.FLAP_HALF_LIFE_MS),
139
+ flapPenaltyMsPerDrop: positiveOr(config.healthCheck?.flapPenaltyMsPerDrop, NODE_MANAGEMENT.FLAP_PENALTY_MS_PER_DROP),
140
+ flapPenaltyMaxMs: positiveOr(config.healthCheck?.flapPenaltyMaxMs, NODE_MANAGEMENT.FLAP_PENALTY_MAX_MS),
104
141
  enabled: config.healthCheck?.enabled ?? true
105
142
  },
106
143
  selection: {
@@ -127,6 +164,7 @@ class NodeManager {
127
164
  });
128
165
  this.initializeNodeStats();
129
166
  this.loadBlacklistState();
167
+ this.loadChurnHistory();
130
168
  // Blacklist warning dedup: track last warn time per node+error to suppress repeats.
131
169
  this._lastBlacklistWarnMs = new Map();
132
170
  // Control flags
@@ -155,10 +193,8 @@ class NodeManager {
155
193
  lastErrorMessage: null,
156
194
  chainId: null,
157
195
  blacklistedAt: null,
158
- // Recovery confirmation: how many consecutive successful
159
- // probes since the last live-transport failure.
160
- healthCheckSuccessStreak: 0,
161
- lastFailureSource: null
196
+ lastFailureAtMs: null,
197
+ dropTimes: []
162
198
  });
163
199
  }
164
200
  }
@@ -182,6 +218,7 @@ class NodeManager {
182
218
  stats.failureCount = typeof entry.failureCount === 'number' ? entry.failureCount : this.config.healthCheck.blacklistThreshold;
183
219
  stats.blacklistedAt = typeof entry.blacklistedAt === 'number' ? entry.blacklistedAt : Date.now();
184
220
  stats.lastErrorMessage = typeof entry.lastErrorMessage === 'string' ? entry.lastErrorMessage : null;
221
+ stats.lastFailureAtMs = typeof entry.lastFailureAtMs === 'number' ? entry.lastFailureAtMs : null;
185
222
  // Seed the shared ledger so counting continues across restarts
186
223
  // (a persisted blacklist must still count toward the threshold
187
224
  // until the cooldown expires or a health check re-admits the
@@ -213,7 +250,8 @@ class NodeManager {
213
250
  status: 'blacklisted',
214
251
  blacklistedAt: stats.blacklistedAt,
215
252
  failureCount: stats.failureCount,
216
- lastErrorMessage: stats.lastErrorMessage
253
+ lastErrorMessage: stats.lastErrorMessage,
254
+ lastFailureAtMs: stats.lastFailureAtMs
217
255
  };
218
256
  }
219
257
  }
@@ -226,6 +264,44 @@ class NodeManager {
226
264
  this.logger.warn(`Failed to save blacklist state: ${getErrorMessage(err)}`);
227
265
  }
228
266
  }
267
+ /**
268
+ * Seed the in-memory drop history from the shared health cache.
269
+ *
270
+ * Churn has to survive process restarts: a bot that is restarted every few
271
+ * hours would otherwise start every node at zero drops and re-select the
272
+ * flapping endpoint immediately. The cache is written by whichever process
273
+ * last ran a health sweep, so treat it as advisory (any parse/age problem
274
+ * just means "no history yet").
275
+ * @private
276
+ */
277
+ loadChurnHistory() {
278
+ try {
279
+ const cache = readHealthCache({
280
+ healthCacheFile: this.healthCacheFile,
281
+ // Keep history that is newer than the flap window; a 4h probe
282
+ // staleness limit would throw away a valid 24h drop history.
283
+ maxAgeMs: NODE_MANAGEMENT.FLAP_WINDOW_MS,
284
+ });
285
+ if (!cache || !Array.isArray(cache.nodes))
286
+ return;
287
+ const now = Date.now();
288
+ for (const entry of cache.nodes) {
289
+ const stats = entry?.url ? this.nodeStats.get(entry.url) : undefined;
290
+ if (!stats || !Array.isArray(entry.dropTimes) || entry.dropTimes.length === 0)
291
+ continue;
292
+ const flapOptions = this._flapOptions(now);
293
+ stats.dropTimes = pruneDropTimes(entry.dropTimes, now, flapOptions.windowMs);
294
+ if (stats.dropTimes.length > 0) {
295
+ const counters = dropCounters(stats.dropTimes, flapOptions);
296
+ this.logger.debug(`Loaded churn history for ${entry.url.substring(0, 40)}... ` +
297
+ `(${counters.drops24h} drop(s)/24h, penalty ~${counters.penaltyMs}ms)`);
298
+ }
299
+ }
300
+ }
301
+ catch (err) {
302
+ this.logger.debug(`Churn history unavailable: ${getErrorMessage(err)}`);
303
+ }
304
+ }
229
305
  /**
230
306
  * Persist the latest healthy/slow node ordering for lightweight consumers
231
307
  * such as the credential daemon.
@@ -233,7 +309,12 @@ class NodeManager {
233
309
  */
234
310
  saveHealthCache() {
235
311
  try {
236
- writeHealthCache(this.nodeStats.values(), { healthCacheFile: this.healthCacheFile });
312
+ writeHealthCache(this.nodeStats.values(), {
313
+ healthCacheFile: this.healthCacheFile,
314
+ // Pass this manager's tuning so the persisted drops/h counters
315
+ // and ordering match what the bot itself would compute.
316
+ flap: this._flapOptions(),
317
+ });
237
318
  }
238
319
  catch (err) {
239
320
  this.logger.warn(`Failed to save node health cache: ${getErrorMessage(err)}`);
@@ -302,6 +383,10 @@ class NodeManager {
302
383
  stats.status = 'unchecked';
303
384
  stats.failureCount = 0;
304
385
  stats.blacklistedAt = null;
386
+ stats.lastFailureAtMs = null;
387
+ // Churn history is intentionally kept: a node coming off
388
+ // blacklist still has to prove stability, and the flap
389
+ // penalty is what keeps it behind a node that never drops.
305
390
  this.failureLedger.reset(nodeUrl);
306
391
  this.saveBlacklistState();
307
392
  this.logger.info(`${nodeUrl.substring(0, 40)}... blacklist cooldown expired, re-enabling`);
@@ -375,41 +460,66 @@ class NodeManager {
375
460
  }
376
461
  }
377
462
  /**
378
- * Record a successful health-probe result: set status/latency and decide
379
- * whether to clear the failure ledger.
463
+ * Remaining (decayed) strikes for a node at `nowMs`.
464
+ *
465
+ * A strike is worth 1.0 when recorded and falls linearly to 0 across
466
+ * `strikeDecayMs`. Linear decay (not exponential) keeps the threshold
467
+ * meaningful: three strikes on the same day still blacklist, while a node
468
+ * that failed once three days ago starts from 0 again. The remainder is
469
+ * rounded *up*, so a fresh single strike is not silently discounted to 0
470
+ * within minutes by the rounding step.
471
+ */
472
+ _decayedStrikeCount(stats, nowMs = Date.now()) {
473
+ if (stats.failureCount <= 0)
474
+ return 0;
475
+ if (!stats.lastFailureAtMs)
476
+ return stats.failureCount;
477
+ const decayMs = this.config.healthCheck.strikeDecayMs;
478
+ const age = Math.max(0, nowMs - stats.lastFailureAtMs);
479
+ if (age >= decayMs)
480
+ return 0;
481
+ const remainingFactor = 1 - age / decayMs;
482
+ return Math.max(0, Math.ceil(stats.failureCount * remainingFactor));
483
+ }
484
+ /**
485
+ * Record a successful health-probe result: set status/latency and let the
486
+ * strike count decay.
380
487
  *
381
- * A single successful probe must not erase strikes that came from the live
382
- * transport. Require `liveFailureSuccessStreak` consecutive successes before
383
- * clearing those, so a genuinely flapping node keeps accumulating toward
384
- * BLACKLIST_THRESHOLD while a recovered node is still re-admitted (status is
385
- * set either way). Strikes whose last source was a health check clear on the
386
- * first success, preserving normal probe recovery.
488
+ * This deliberately no longer *resets* strikes: the old
489
+ * liveFailureSuccessStreak rule cleared a live-transport strike after two
490
+ * successful probes, and with a 4h probe interval that forgave a
491
+ * once-a-day flapper twice a day — every failure logged "attempt 1/3" and
492
+ * the node was never blacklisted. Strikes now age out on their own
493
+ * (`strikeDecayMs`), so probe success can re-admit a node's *status*
494
+ * without erasing the evidence that it drops connections.
387
495
  *
388
- * @returns {{ recovered: boolean, failureCount: number }}
496
+ * @returns {{ recovered: boolean, failureCount: number }} whether every
497
+ * strike has aged out, and the remaining count.
389
498
  */
390
499
  _recordHealthCheckSuccess(nodeUrl, status, latencyMs, chainId) {
391
500
  const stats = this.nodeStats.get(nodeUrl);
392
501
  if (!stats)
393
502
  return { recovered: false, failureCount: 0 };
394
- const priorSource = stats.lastFailureSource;
395
- const liveFailure = !!priorSource && priorSource !== 'health-check';
396
- stats.healthCheckSuccessStreak = (stats.healthCheckSuccessStreak || 0) + 1;
397
- const successStreak = this.config.healthCheck.liveFailureSuccessStreak;
398
- const recovered = !liveFailure || stats.healthCheckSuccessStreak >= successStreak;
399
503
  stats.status = status;
400
504
  stats.latencyMs = latencyMs;
401
505
  stats.lastCheckTime = nowIso();
402
506
  stats.lastErrorMessage = null;
403
507
  stats.chainId = chainId;
404
- if (recovered) {
405
- stats.failureCount = 0;
406
- stats.lastFailureSource = null;
407
- stats.healthCheckSuccessStreak = 0;
408
- this.failureLedger.reset(nodeUrl);
409
- }
410
- else {
411
- this.logger.debug(`${nodeUrl.substring(0, 40)}... ${status} but retaining ${stats.failureCount} live-transport strike(s) ` +
412
- `(success streak ${stats.healthCheckSuccessStreak}/${successStreak})`);
508
+ const remaining = this._decayedStrikeCount(stats);
509
+ const recovered = remaining <= 0;
510
+ if (stats.failureCount > 0) {
511
+ if (remaining <= 0) {
512
+ stats.failureCount = 0;
513
+ stats.lastFailureAtMs = null;
514
+ this.failureLedger.reset(nodeUrl);
515
+ }
516
+ else {
517
+ stats.failureCount = remaining;
518
+ // Keep the ledger and the visible count in step, otherwise the
519
+ // next failure would blacklist on stale evidence.
520
+ this.failureLedger.seed(nodeUrl, { failureCount: remaining, blacklistedUntil: 0 });
521
+ this.logger.debug(`${nodeUrl.substring(0, 40)}... ${status} but retaining ${remaining} decaying strike(s)`);
522
+ }
413
523
  }
414
524
  return { recovered, failureCount: stats.failureCount };
415
525
  }
@@ -567,13 +677,40 @@ class NodeManager {
567
677
  });
568
678
  }
569
679
  /**
570
- * Get list of healthy nodes sorted by latency (best first)
680
+ * Flap-scoring options built from this manager's configuration.
681
+ * Single place that maps config -> the pure ranking maths, so the ranking,
682
+ * the logged counters and the persisted health cache always agree.
683
+ * @private
684
+ */
685
+ _flapOptions(nowMs = Date.now()) {
686
+ return {
687
+ nowMs,
688
+ windowMs: NODE_MANAGEMENT.FLAP_WINDOW_MS,
689
+ halfLifeMs: this.config.healthCheck.flapHalfLifeMs,
690
+ penaltyMsPerDrop: this.config.healthCheck.flapPenaltyMsPerDrop,
691
+ maxPenaltyMs: this.config.healthCheck.flapPenaltyMaxMs,
692
+ };
693
+ }
694
+ /**
695
+ * Get list of healthy nodes, best first.
696
+ *
697
+ * Ranking is `measured latency + decayed connection-churn penalty`, so a
698
+ * 50ms node that drops its socket daily loses to a stable 350ms node. See
699
+ * modules/node_flap_score.ts for the penalty maths.
571
700
  * @returns {string[]} Array of node URLs
572
701
  */
573
702
  getHealthyNodes() {
574
703
  const preferredNode = this.config.selection.preferredNode;
575
- const healthy = Array.from(this.nodeStats.values())
576
- .filter((stat) => stat.status === 'healthy' || stat.status === 'slow')
704
+ const flapOptions = this._flapOptions();
705
+ const healthyStats = Array.from(this.nodeStats.values())
706
+ .filter((stat) => stat.status === 'healthy' || stat.status === 'slow');
707
+ // Precompute one rank per node: doing it inside the comparator would
708
+ // prune and re-sort every node's drop history O(n log n) times.
709
+ const ranks = new Map();
710
+ for (const stat of healthyStats) {
711
+ ranks.set(stat, effectiveLatencyMs(stat.latencyMs, stat.dropTimes, flapOptions));
712
+ }
713
+ const healthy = healthyStats
577
714
  .sort((a, b) => {
578
715
  if (preferredNode) {
579
716
  const aPreferred = a.url === preferredNode && a.status === 'healthy';
@@ -581,13 +718,36 @@ class NodeManager {
581
718
  if (aPreferred !== bPreferred)
582
719
  return aPreferred ? -1 : 1;
583
720
  }
584
- return compareNodeHealth(a, b);
721
+ if (a.status !== b.status)
722
+ return a.status === 'healthy' ? -1 : 1;
723
+ // Compare without subtracting: two unknown latencies are both
724
+ // Infinity, and Infinity - Infinity is NaN (an unstable sort
725
+ // comparator).
726
+ const aRank = ranks.get(a) ?? Infinity;
727
+ const bRank = ranks.get(b) ?? Infinity;
728
+ if (aRank === bRank)
729
+ return 0;
730
+ return aRank < bRank ? -1 : 1;
585
731
  })
586
732
  .map((stat) => stat.url);
587
733
  return healthy;
588
734
  }
589
735
  /**
590
- * Get the single best (lowest latency) healthy node
736
+ * Connection-churn counters for one node (drops/h, decayed score, penalty).
737
+ * Used for logging and for the shared health cache.
738
+ * @param {string} nodeUrl - Node URL
739
+ * @returns {Object|null} Churn counters or null for an unknown node
740
+ */
741
+ getNodeChurn(nodeUrl, nowMs = Date.now()) {
742
+ const stats = this.nodeStats.get(nodeUrl);
743
+ if (!stats)
744
+ return null;
745
+ const flapOptions = this._flapOptions(nowMs);
746
+ const dropTimes = pruneDropTimes(stats.dropTimes, nowMs, flapOptions.windowMs);
747
+ return { dropTimes, ...dropCounters(dropTimes, flapOptions) };
748
+ }
749
+ /**
750
+ * Get the single best-ranked healthy node (latency + churn penalty)
591
751
  * @returns {string|null} Best node URL or null if none available
592
752
  */
593
753
  getBestNode() {
@@ -621,41 +781,55 @@ class NodeManager {
621
781
  * failure ledger — the same rules the credential daemon applies.
622
782
  * @param {string} nodeUrl - Node URL that failed
623
783
  * @param {string} [errorMessage] - Optional error description
624
- * @param {string} [source] - Failure source ('health-check' or 'broadcast'); controls log level
784
+ * @param {string} [source] - Failure source ('health-check', 'broadcast',
785
+ * 'connection', 'keep-alive'); controls log level and churn accounting
786
+ * @param {number} [nowMs] - Timestamp for the strike/drop (defaults to now;
787
+ * injectable so tests do not need to sleep past the report cooldown)
625
788
  */
626
- reportNodeFailure(nodeUrl, errorMessage, source) {
789
+ reportNodeFailure(nodeUrl, errorMessage, source, nowMs = Date.now()) {
627
790
  const stats = this.nodeStats.get(nodeUrl);
628
791
  if (!stats) {
629
792
  this.logger.warn(`Node ${nodeUrl} not in configured list, cannot report failure`);
630
793
  return;
631
794
  }
632
795
  const isHealthCheck = source === 'health-check';
633
- const result = this.failureLedger.recordFailure(nodeUrl);
796
+ const result = this.failureLedger.recordFailure(nodeUrl, nowMs);
634
797
  stats.failureCount = result.failureCount;
635
798
  stats.lastCheckTime = nowIso();
636
799
  if (errorMessage)
637
800
  stats.lastErrorMessage = errorMessage;
638
- // Remember the source so a later health-check success knows whether the
639
- // strike came from the live transport (retain) or a probe (clearable).
640
- // Recorded even when the ledger rate-limits this report.
641
- stats.lastFailureSource = source || 'unknown';
642
- stats.healthCheckSuccessStreak = 0;
801
+ const counted = result.outcome === 'counted' || result.outcome === 'blacklisted';
802
+ // A counted strike refreshes the decay clock. Recording it even for
803
+ // probe failures keeps `failureCount` and its age consistent.
804
+ if (counted) {
805
+ stats.lastFailureAtMs = nowMs;
806
+ }
807
+ // Only live-operation failures feed the churn score used by selection
808
+ // (see CHURN_SOURCES). Rate-limited reports restate the same event, so
809
+ // they are excluded along with blacklisted-node repeats.
810
+ if (counted && source && CHURN_SOURCES.has(source)) {
811
+ stats.dropTimes = recordDrop(stats.dropTimes, nowMs, this._flapOptions(nowMs).windowMs);
812
+ }
643
813
  if (result.outcome === 'rate-limited' || result.outcome === 'skipped-blacklisted') {
644
814
  return;
645
815
  }
646
816
  if (result.outcome === 'blacklisted') {
647
817
  stats.status = 'blacklisted';
648
- stats.blacklistedAt = Date.now();
818
+ stats.blacklistedAt = nowMs;
649
819
  this.saveBlacklistState();
650
820
  this.saveHealthCache();
651
821
  if (this._shouldLogBlacklistWarning(nodeUrl, errorMessage || '')) {
652
- this.logger.warn(`✗ ${nodeUrl.substring(0, 40)}... BLACKLISTED after ${result.failureCount} failures (${errorMessage || 'unknown'})`);
822
+ const churn = this.getNodeChurn(nodeUrl);
823
+ this.logger.warn(`✗ ${nodeUrl.substring(0, 40)}... BLACKLISTED after ${result.failureCount} failures (${errorMessage || 'unknown'}) ` +
824
+ `[churn ${churn ? `${churn.drops24h} drop(s)/24h, +${churn.penaltyMs}ms` : 'n/a'}]`);
653
825
  }
654
826
  }
655
827
  else {
656
828
  stats.status = 'failed';
657
829
  const log = isHealthCheck ? this.logger.debug.bind(this.logger) : this.logger.warn.bind(this.logger);
658
- log(`⚠ ${nodeUrl.substring(0, 40)}... FAILED attempt ${result.failureCount}/${result.threshold} (${errorMessage || 'unknown'})`);
830
+ const churn = this.getNodeChurn(nodeUrl);
831
+ log(`⚠ ${nodeUrl.substring(0, 40)}... FAILED attempt ${result.failureCount}/${result.threshold} (${errorMessage || 'unknown'})` +
832
+ (churn && churn.drops24h > 0 ? ` [churn ${churn.drops24h} drop(s)/24h, +${churn.penaltyMs}ms]` : ''));
659
833
  }
660
834
  }
661
835
  /**
@@ -668,6 +842,7 @@ class NodeManager {
668
842
  stats.status = 'blacklisted';
669
843
  stats.failureCount = this.config.healthCheck.blacklistThreshold;
670
844
  stats.blacklistedAt = Date.now();
845
+ stats.lastFailureAtMs = stats.blacklistedAt;
671
846
  this.failureLedger.seed(nodeUrl, {
672
847
  failureCount: stats.failureCount,
673
848
  blacklistedUntil: stats.blacklistedAt + this.BLACKLIST_COOLDOWN_MS,
@@ -689,8 +864,10 @@ class NodeManager {
689
864
  stats.latencyMs = null;
690
865
  stats.lastErrorMessage = null;
691
866
  stats.blacklistedAt = null;
692
- stats.lastFailureSource = null;
693
- stats.healthCheckSuccessStreak = 0;
867
+ stats.lastFailureAtMs = null;
868
+ // An explicit reset is the operator saying "give this node a clean
869
+ // slate": churn history goes with the strikes.
870
+ stats.dropTimes = [];
694
871
  this.failureLedger.reset(nodeUrl);
695
872
  this._clearBlacklistWarningCooldown(nodeUrl);
696
873
  this.saveBlacklistState();
@@ -709,8 +886,8 @@ class NodeManager {
709
886
  stats.lastErrorMessage = null;
710
887
  stats.chainId = null;
711
888
  stats.blacklistedAt = null;
712
- stats.lastFailureSource = null;
713
- stats.healthCheckSuccessStreak = 0;
889
+ stats.lastFailureAtMs = null;
890
+ stats.dropTimes = [];
714
891
  }
715
892
  this.failureLedger.clear();
716
893
  this._clearBlacklistWarningCooldown();
@@ -723,14 +900,21 @@ class NodeManager {
723
900
  * @returns {Array<Object>} Array of node stats
724
901
  */
725
902
  getStats() {
726
- return Array.from(this.nodeStats.values()).map((stat) => ({
727
- url: stat.url,
728
- status: stat.status,
729
- latencyMs: stat.latencyMs,
730
- failureCount: stat.failureCount,
731
- lastCheckTime: stat.lastCheckTime,
732
- lastErrorMessage: stat.lastErrorMessage
733
- }));
903
+ const nowMs = Date.now();
904
+ return Array.from(this.nodeStats.values()).map((stat) => {
905
+ const churn = this.getNodeChurn(stat.url, nowMs);
906
+ return {
907
+ url: stat.url,
908
+ status: stat.status,
909
+ latencyMs: stat.latencyMs,
910
+ failureCount: stat.failureCount,
911
+ lastCheckTime: stat.lastCheckTime,
912
+ lastErrorMessage: stat.lastErrorMessage,
913
+ drops24h: churn ? churn.drops24h : 0,
914
+ dropsPerHour: churn ? churn.dropsPerHour : 0,
915
+ flapPenaltyMs: churn ? churn.penaltyMs : 0
916
+ };
917
+ });
734
918
  }
735
919
  /**
736
920
  * Get summary statistics
@@ -755,11 +939,17 @@ class NodeManager {
755
939
  const avgLatency = statsWithLatency.length > 0
756
940
  ? Math.round(statsWithLatency.reduce((sum, s) => sum + s.latencyMs, 0) / statsWithLatency.length)
757
941
  : null;
942
+ const nowMs = Date.now();
943
+ const totalDrops24h = stats.reduce((sum, s) => {
944
+ const churn = this.getNodeChurn(s.url, nowMs);
945
+ return sum + (churn ? churn.drops24h : 0);
946
+ }, 0);
758
947
  return {
759
948
  monitoring: this.monitoringActive,
760
949
  counts,
761
950
  bestNode,
762
- avgLatency
951
+ avgLatency,
952
+ totalDrops24h
763
953
  };
764
954
  }
765
955
  }
@@ -6,7 +6,7 @@
6
6
  * Exports a single Accountant class that manages all fund accounting operations.
7
7
  *
8
8
  * ===============================================================================
9
- * TABLE OF CONTENTS - Accountant Class (26 methods)
9
+ * TABLE OF CONTENTS - Accountant Class (27 methods)
10
10
  * ===============================================================================
11
11
  *
12
12
  * CORE INITIALIZATION & RECALCULATION (2 methods)
@@ -56,6 +56,7 @@
56
56
  * 25. _recalibrateTrackedFundsFromChain(mgr, openOrders) - Recalibrate tracked funds
57
57
  * from a fresh chain snapshot
58
58
  * 26. _tryTrustChainFreeHeal(mgr, driftValidation) - Attempt a trust-chain-free heal
59
+ * 27. _getBtsFeeSchedule() - Resolve the BTS fee schedule (create/update/cancel + maker refund %)
59
60
  *
60
61
  * ===============================================================================
61
62
  * FUND STRUCTURE (managed by Accountant)
@@ -131,9 +132,13 @@ declare class Accountant {
131
132
  constructor(manager: OrderManagerLike);
132
133
  _logThrottled(key: string, message: string, level?: string, intervalMs?: number): boolean;
133
134
  /**
134
- * Apply a list of balance adjustments.
135
- * @param {Array<{orderType: string, delta: number, operation: string}>} balanceAdjustments
136
- * @returns {void}
135
+ * Apply balance adjustments ALL-OR-NOTHING.
136
+ *
137
+ * A throw from the middle of the list used to leave the earlier deltas
138
+ * applied. Since P2 releases the dedup claim on failure (so the fill can be
139
+ * retried), a partial application would then be applied twice on the retry.
140
+ * The credit is now atomic: an applied prefix is reversed before the error
141
+ * propagates, so "not credited" is true whenever the claim was released.
137
142
  */
138
143
  _applyBalanceAdjustments(balanceAdjustments: Array<{
139
144
  orderType: string | null | undefined;