dexbot 1.6.18 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/CHANGELOG.md +68 -0
  2. package/README.md +24 -18
  3. package/analysis/README.md +117 -5
  4. package/claw/package.json +1 -1
  5. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  6. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  7. package/dist/analysis/account_resolver.js +5 -6
  8. package/dist/analysis/analyze_dynamic_weight.js +2 -1
  9. package/dist/analysis/analyze_kalman.js +3 -2
  10. package/dist/analysis/analyze_liquidity.d.ts +17 -0
  11. package/dist/analysis/analyze_liquidity.js +336 -0
  12. package/dist/analysis/analyze_regime.js +3 -2
  13. package/dist/analysis/analyze_regime_windows.js +3 -2
  14. package/dist/analysis/analyze_volatility.js +3 -2
  15. package/dist/analysis/bot_key_utils.d.ts +44 -1
  16. package/dist/analysis/bot_key_utils.js +76 -7
  17. package/dist/analysis/fills_cache.js +2 -1
  18. package/dist/analysis/grid_correction_check.d.ts +1 -19
  19. package/dist/analysis/grid_correction_check.js +12 -30
  20. package/dist/analysis/liquidity/liquidity_chain.d.ts +66 -0
  21. package/dist/analysis/liquidity/liquidity_chain.js +178 -0
  22. package/dist/analysis/liquidity/liquidity_chart_generator.d.ts +19 -0
  23. package/dist/analysis/liquidity/liquidity_chart_generator.js +693 -0
  24. package/dist/analysis/liquidity/liquidity_depth_model.d.ts +166 -0
  25. package/dist/analysis/liquidity/liquidity_depth_model.js +258 -0
  26. package/dist/analysis/liquidity/liquidity_view.d.ts +53 -0
  27. package/dist/analysis/liquidity/liquidity_view.js +66 -0
  28. package/dist/analysis/resolve_bot_accounts.js +7 -6
  29. package/dist/analysis/trade_profitability.js +4 -2
  30. package/dist/analysis/tradingview/analyze_tradingview.js +2 -1
  31. package/dist/credential-daemon.js +13 -2
  32. package/dist/dexbot.js +77 -172
  33. package/dist/modules/account_bots.d.ts +11 -1
  34. package/dist/modules/account_orders.d.ts +12 -0
  35. package/dist/modules/account_orders.js +30 -0
  36. package/dist/modules/asset_lookup.d.ts +25 -0
  37. package/dist/modules/asset_lookup.js +75 -0
  38. package/dist/modules/bitshares-native/transport.js +24 -6
  39. package/dist/modules/bitshares_client.d.ts +24 -6
  40. package/dist/modules/bitshares_client.js +44 -7
  41. package/dist/modules/bot_settings.d.ts +2 -0
  42. package/dist/modules/chain_scan.d.ts +64 -0
  43. package/dist/modules/chain_scan.js +80 -0
  44. package/dist/modules/cli_colors.d.ts +5 -0
  45. package/dist/modules/cli_colors.js +7 -0
  46. package/dist/modules/cli_progress.d.ts +51 -0
  47. package/dist/modules/cli_progress.js +113 -0
  48. package/dist/modules/constants.d.ts +26 -3
  49. package/dist/modules/constants.js +63 -8
  50. package/dist/modules/dexbot_class.d.ts +1 -6
  51. package/dist/modules/dexbot_class.js +14 -0
  52. package/dist/modules/dexbot_cow_runtime.d.ts +97 -7
  53. package/dist/modules/dexbot_cow_runtime.js +634 -136
  54. package/dist/modules/dexbot_fill_runtime.js +132 -23
  55. package/dist/modules/dexbot_maintenance_runtime.js +242 -74
  56. package/dist/modules/dexbot_startup_runtime.js +32 -20
  57. package/dist/modules/dexbot_state_recovery.d.ts +33 -1
  58. package/dist/modules/dexbot_state_recovery.js +167 -5
  59. package/dist/modules/launcher/runtime_entry.d.ts +21 -1
  60. package/dist/modules/launcher/runtime_entry.js +36 -1
  61. package/dist/modules/liquidity_pool_ref.d.ts +32 -0
  62. package/dist/modules/liquidity_pool_ref.js +116 -0
  63. package/dist/modules/node_connect_policy.d.ts +8 -1
  64. package/dist/modules/node_connect_policy.js +18 -2
  65. package/dist/modules/node_flap_score.d.ts +87 -0
  66. package/dist/modules/node_flap_score.js +124 -0
  67. package/dist/modules/node_health_cache.d.ts +40 -3
  68. package/dist/modules/node_health_cache.js +66 -20
  69. package/dist/modules/node_manager.d.ts +101 -19
  70. package/dist/modules/node_manager.js +255 -65
  71. package/dist/modules/order/accounting.js +56 -7
  72. package/dist/modules/order/format.d.ts +24 -10
  73. package/dist/modules/order/format.js +30 -10
  74. package/dist/modules/order/grid.js +91 -13
  75. package/dist/modules/order/grid_reconcile.d.ts +5 -1
  76. package/dist/modules/order/grid_reconcile.js +333 -189
  77. package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
  78. package/dist/modules/order/grid_reconcile_internal.js +298 -100
  79. package/dist/modules/order/index.d.ts +19 -1
  80. package/dist/modules/order/lifecycle.d.ts +353 -0
  81. package/dist/modules/order/lifecycle.js +1109 -0
  82. package/dist/modules/order/manager.d.ts +273 -5
  83. package/dist/modules/order/manager.js +654 -48
  84. package/dist/modules/order/placement_lock.d.ts +102 -0
  85. package/dist/modules/order/placement_lock.js +222 -0
  86. package/dist/modules/order/strategy.js +13 -1
  87. package/dist/modules/order/sync_engine.d.ts +33 -1
  88. package/dist/modules/order/sync_engine.js +241 -73
  89. package/dist/modules/order/utils/math.d.ts +39 -1
  90. package/dist/modules/order/utils/math.js +83 -1
  91. package/dist/modules/order/utils/order.d.ts +181 -5
  92. package/dist/modules/order/utils/order.js +455 -9
  93. package/dist/modules/order/utils/system.d.ts +81 -1
  94. package/dist/modules/order/utils/system.js +331 -44
  95. package/dist/modules/order/utils/validate.d.ts +7 -1
  96. package/dist/modules/order/utils/validate.js +80 -3
  97. package/dist/modules/outcome.d.ts +54 -0
  98. package/dist/modules/outcome.js +51 -0
  99. package/dist/modules/settings_merge.d.ts +4 -0
  100. package/dist/modules/settings_merge.js +9 -0
  101. package/dist/modules/timing_policy.d.ts +82 -0
  102. package/dist/modules/timing_policy.js +244 -0
  103. package/dist/modules/types.d.ts +137 -0
  104. package/dist/modules/utils/asset_symbols.d.ts +13 -2
  105. package/dist/modules/utils/asset_symbols.js +21 -2
  106. package/dist/scripts/analysis_runner.d.ts +10 -0
  107. package/dist/scripts/analysis_runner.js +33 -0
  108. package/dist/scripts/analyze-credit.js +54 -17
  109. package/dist/scripts/analyze-orders.d.ts +4 -2
  110. package/dist/scripts/analyze-orders.js +115 -35
  111. package/dist/scripts/asset.d.ts +26 -0
  112. package/dist/scripts/asset.js +165 -0
  113. package/dist/scripts/bts_distribution.d.ts +1 -0
  114. package/dist/scripts/bts_distribution.js +194 -142
  115. package/dist/scripts/chart_command.d.ts +1 -1
  116. package/dist/scripts/chart_command.js +60 -86
  117. package/dist/scripts/chart_targets.d.ts +18 -0
  118. package/dist/scripts/chart_targets.js +48 -0
  119. package/dist/scripts/dw.js +1 -1
  120. package/dist/scripts/liquid.d.ts +24 -0
  121. package/dist/scripts/liquid.js +134 -0
  122. package/dist/scripts/rich_list.js +12 -16
  123. package/dist/scripts/test-credit-renewal.js +12 -5
  124. package/dist/scripts/token_distribution.d.ts +2 -0
  125. package/dist/scripts/token_distribution.js +132 -108
  126. package/dist/scripts/tv.js +1 -1
  127. package/dist/scripts/verify-browser-bundle.js +18 -0
  128. package/docs/COW_INVARIANTS.md +69 -10
  129. package/docs/DEXBOT_COMPARISON.md +6 -6
  130. package/docs/EVOLUTION.md +10 -8
  131. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  132. package/docs/GRID_PRICE_INVARIANT.md +5 -2
  133. package/docs/GRID_RECONCILE.md +36 -11
  134. package/docs/LOGGING.md +50 -4
  135. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +177 -27
  136. package/docs/README.md +23 -2
  137. package/docs/WORKFLOW.md +12 -4
  138. package/docs/architecture.md +42 -5
  139. package/docs/developer_guide.md +25 -8
  140. package/modules/README.md +8 -1
  141. package/package.json +3 -1
  142. package/scripts/README.md +39 -7
  143. package/scripts/pdev.sh +14 -1
  144. package/scripts/pmain.sh +23 -8
  145. package/scripts/ptest.sh +7 -0
  146. package/tests/README.md +30 -1
@@ -15,7 +15,9 @@
15
15
  * - Automatic blacklisting of failed nodes
16
16
  *
17
17
  * 2. Intelligent Node Selection
18
- * - Selects lowest-latency healthy node
18
+ * - Ranks nodes by measured latency plus a time-decayed connection-churn
19
+ * penalty (see modules/node_flap_score.ts), so a fast node that keeps
20
+ * dropping its socket loses to a stable slightly-slower one
19
21
  * - Falls back to slow nodes if no healthy ones available
20
22
  * - Respects user-preferred node if healthy
21
23
  *
@@ -44,19 +46,49 @@ const require = createRequire(import.meta.url);
44
46
  import Logger from './order/logger.js';
45
47
  import { NODE_MANAGEMENT } from './constants.js';
46
48
  import { PATHS, getNodeBlacklistFile } from './paths.js';
47
- import { compareNodeHealth } from './node_health_cache.js';
48
49
  import { writeJsonFileAtomic } from './bots_file_lock.js';
49
50
  import { getStorage } from './storage/index.js';
50
51
  const storage = getStorage();
51
52
  const { readJSON } = storage;
52
53
  import { createFailureLedger } from './node_failure_ledger.js';
53
54
  const _WebSocket = globalThis.WebSocket;
54
- import { resolveHealthCacheFile, writeHealthCache, } from './node_health_cache.js';
55
+ import { readHealthCache, resolveHealthCacheFile, writeHealthCache, } from './node_health_cache.js';
56
+ import { dropCounters, effectiveLatencyMs, pruneDropTimes, recordDrop, } from './node_flap_score.js';
55
57
  import { getErrorMessage, getErrorCode } from './utils/errors.js';
56
58
  import { nowIso } from './order/utils/system.js';
59
+ /**
60
+ * Failure sources that count as a live-operation reliability event for churn
61
+ * scoring/selection.
62
+ *
63
+ * Included:
64
+ * - `connection` an unexpected socket close (code 1006/1001 etc.)
65
+ * - `keep-alive` the socket stayed open but stopped answering
66
+ * - `broadcast` a signed batch was accepted but never confirmed in time
67
+ * (BROADCAST_DEADLINE) — the socket is up, the node is not
68
+ * serving us, which is exactly the instability selection
69
+ * should penalize
70
+ *
71
+ * Excluded:
72
+ * - `health-check` probe-side noise; it already moves the node out of the
73
+ * healthy set, and probes may fail for reasons that never
74
+ * affect a live session
75
+ * - `forced-reconnect` a client-side recovery attempt, not evidence that the
76
+ * node is bad (same rule as shouldCountNodeStrike in
77
+ * bitshares_client)
78
+ */
79
+ const CHURN_SOURCES = new Set(['connection', 'keep-alive', 'broadcast']);
80
+ /** Positive finite number or the fallback (config may carry 0/NaN/strings). */
81
+ function positiveOr(value, fallback) {
82
+ const n = Number(value);
83
+ return Number.isFinite(n) && n > 0 ? n : fallback;
84
+ }
57
85
  /**
58
86
  * NodeManager - Manages health checking and selection of BitShares nodes
59
87
  *
88
+ * Selection ranks healthy nodes on `latency + decayed churn penalty`, keeps a
89
+ * time-decaying strike count for blacklisting, and exposes per-node churn
90
+ * counters (drops/h) for logging and the shared health cache.
91
+ *
60
92
  * @class
61
93
  * @param {Object} config - Configuration object
62
94
  * @param {string[]} config.list - List of node URLs to monitor (e.g., wss://dex.iobanker.com/ws)
@@ -65,8 +97,12 @@ import { nowIso } from './order/utils/system.js';
65
97
  * @param {number} [config.healthCheck.timeoutMs=5000] - Timeout for each health check
66
98
  * @param {number} [config.healthCheck.maxPingMs=3000] - Max acceptable latency
67
99
  * @param {number} [config.healthCheck.blacklistThreshold=3] - Failures before blacklist
100
+ * @param {number} [config.healthCheck.strikeDecayMs] - Lifetime of a strike without a new failure
101
+ * @param {number} [config.healthCheck.flapHalfLifeMs] - Half-life of a connection-drop penalty
102
+ * @param {number} [config.healthCheck.flapPenaltyMsPerDrop] - Effective-latency cost per fresh drop
103
+ * @param {number} [config.healthCheck.flapPenaltyMaxMs] - Cap on the decayed churn penalty
68
104
  * @param {Object} [config.selection] - Node selection settings
69
- * @param {string} [config.selection.strategy="latency"] - Selection strategy
105
+ * @param {string} [config.selection.strategy="latency"] - Selection strategy (latency + churn penalty)
70
106
  * @param {string} [config.selection.preferredNode] - Optional preferred node (if healthy)
71
107
  */
72
108
  class NodeManager {
@@ -98,9 +134,10 @@ class NodeManager {
98
134
  timeoutMs: config.healthCheck?.timeoutMs ?? NODE_MANAGEMENT.HEALTH_CHECK_TIMEOUT_MS,
99
135
  maxPingMs: config.healthCheck?.maxPingMs ?? NODE_MANAGEMENT.MAX_PING_MS,
100
136
  blacklistThreshold: config.healthCheck?.blacklistThreshold ?? NODE_MANAGEMENT.BLACKLIST_THRESHOLD,
101
- liveFailureSuccessStreak: Number(config.healthCheck?.liveFailureSuccessStreak) > 0
102
- ? Number(config.healthCheck?.liveFailureSuccessStreak)
103
- : NODE_MANAGEMENT.LIVE_FAILURE_HEALTH_SUCCESS_STREAK,
137
+ strikeDecayMs: positiveOr(config.healthCheck?.strikeDecayMs, NODE_MANAGEMENT.FAILURE_STRIKE_DECAY_MS),
138
+ flapHalfLifeMs: positiveOr(config.healthCheck?.flapHalfLifeMs, NODE_MANAGEMENT.FLAP_HALF_LIFE_MS),
139
+ flapPenaltyMsPerDrop: positiveOr(config.healthCheck?.flapPenaltyMsPerDrop, NODE_MANAGEMENT.FLAP_PENALTY_MS_PER_DROP),
140
+ flapPenaltyMaxMs: positiveOr(config.healthCheck?.flapPenaltyMaxMs, NODE_MANAGEMENT.FLAP_PENALTY_MAX_MS),
104
141
  enabled: config.healthCheck?.enabled ?? true
105
142
  },
106
143
  selection: {
@@ -127,6 +164,7 @@ class NodeManager {
127
164
  });
128
165
  this.initializeNodeStats();
129
166
  this.loadBlacklistState();
167
+ this.loadChurnHistory();
130
168
  // Blacklist warning dedup: track last warn time per node+error to suppress repeats.
131
169
  this._lastBlacklistWarnMs = new Map();
132
170
  // Control flags
@@ -155,10 +193,8 @@ class NodeManager {
155
193
  lastErrorMessage: null,
156
194
  chainId: null,
157
195
  blacklistedAt: null,
158
- // Recovery confirmation: how many consecutive successful
159
- // probes since the last live-transport failure.
160
- healthCheckSuccessStreak: 0,
161
- lastFailureSource: null
196
+ lastFailureAtMs: null,
197
+ dropTimes: []
162
198
  });
163
199
  }
164
200
  }
@@ -182,6 +218,7 @@ class NodeManager {
182
218
  stats.failureCount = typeof entry.failureCount === 'number' ? entry.failureCount : this.config.healthCheck.blacklistThreshold;
183
219
  stats.blacklistedAt = typeof entry.blacklistedAt === 'number' ? entry.blacklistedAt : Date.now();
184
220
  stats.lastErrorMessage = typeof entry.lastErrorMessage === 'string' ? entry.lastErrorMessage : null;
221
+ stats.lastFailureAtMs = typeof entry.lastFailureAtMs === 'number' ? entry.lastFailureAtMs : null;
185
222
  // Seed the shared ledger so counting continues across restarts
186
223
  // (a persisted blacklist must still count toward the threshold
187
224
  // until the cooldown expires or a health check re-admits the
@@ -213,7 +250,8 @@ class NodeManager {
213
250
  status: 'blacklisted',
214
251
  blacklistedAt: stats.blacklistedAt,
215
252
  failureCount: stats.failureCount,
216
- lastErrorMessage: stats.lastErrorMessage
253
+ lastErrorMessage: stats.lastErrorMessage,
254
+ lastFailureAtMs: stats.lastFailureAtMs
217
255
  };
218
256
  }
219
257
  }
@@ -226,6 +264,44 @@ class NodeManager {
226
264
  this.logger.warn(`Failed to save blacklist state: ${getErrorMessage(err)}`);
227
265
  }
228
266
  }
267
+ /**
268
+ * Seed the in-memory drop history from the shared health cache.
269
+ *
270
+ * Churn has to survive process restarts: a bot that is restarted every few
271
+ * hours would otherwise start every node at zero drops and re-select the
272
+ * flapping endpoint immediately. The cache is written by whichever process
273
+ * last ran a health sweep, so treat it as advisory (any parse/age problem
274
+ * just means "no history yet").
275
+ * @private
276
+ */
277
+ loadChurnHistory() {
278
+ try {
279
+ const cache = readHealthCache({
280
+ healthCacheFile: this.healthCacheFile,
281
+ // Keep history that is newer than the flap window; a 4h probe
282
+ // staleness limit would throw away a valid 24h drop history.
283
+ maxAgeMs: NODE_MANAGEMENT.FLAP_WINDOW_MS,
284
+ });
285
+ if (!cache || !Array.isArray(cache.nodes))
286
+ return;
287
+ const now = Date.now();
288
+ for (const entry of cache.nodes) {
289
+ const stats = entry?.url ? this.nodeStats.get(entry.url) : undefined;
290
+ if (!stats || !Array.isArray(entry.dropTimes) || entry.dropTimes.length === 0)
291
+ continue;
292
+ const flapOptions = this._flapOptions(now);
293
+ stats.dropTimes = pruneDropTimes(entry.dropTimes, now, flapOptions.windowMs);
294
+ if (stats.dropTimes.length > 0) {
295
+ const counters = dropCounters(stats.dropTimes, flapOptions);
296
+ this.logger.debug(`Loaded churn history for ${entry.url.substring(0, 40)}... ` +
297
+ `(${counters.drops24h} drop(s)/24h, penalty ~${counters.penaltyMs}ms)`);
298
+ }
299
+ }
300
+ }
301
+ catch (err) {
302
+ this.logger.debug(`Churn history unavailable: ${getErrorMessage(err)}`);
303
+ }
304
+ }
229
305
  /**
230
306
  * Persist the latest healthy/slow node ordering for lightweight consumers
231
307
  * such as the credential daemon.
@@ -233,7 +309,12 @@ class NodeManager {
233
309
  */
234
310
  saveHealthCache() {
235
311
  try {
236
- writeHealthCache(this.nodeStats.values(), { healthCacheFile: this.healthCacheFile });
312
+ writeHealthCache(this.nodeStats.values(), {
313
+ healthCacheFile: this.healthCacheFile,
314
+ // Pass this manager's tuning so the persisted drops/h counters
315
+ // and ordering match what the bot itself would compute.
316
+ flap: this._flapOptions(),
317
+ });
237
318
  }
238
319
  catch (err) {
239
320
  this.logger.warn(`Failed to save node health cache: ${getErrorMessage(err)}`);
@@ -302,6 +383,10 @@ class NodeManager {
302
383
  stats.status = 'unchecked';
303
384
  stats.failureCount = 0;
304
385
  stats.blacklistedAt = null;
386
+ stats.lastFailureAtMs = null;
387
+ // Churn history is intentionally kept: a node coming off
388
+ // blacklist still has to prove stability, and the flap
389
+ // penalty is what keeps it behind a node that never drops.
305
390
  this.failureLedger.reset(nodeUrl);
306
391
  this.saveBlacklistState();
307
392
  this.logger.info(`${nodeUrl.substring(0, 40)}... blacklist cooldown expired, re-enabling`);
@@ -375,41 +460,66 @@ class NodeManager {
375
460
  }
376
461
  }
377
462
  /**
378
- * Record a successful health-probe result: set status/latency and decide
379
- * whether to clear the failure ledger.
463
+ * Remaining (decayed) strikes for a node at `nowMs`.
464
+ *
465
+ * A strike is worth 1.0 when recorded and falls linearly to 0 across
466
+ * `strikeDecayMs`. Linear decay (not exponential) keeps the threshold
467
+ * meaningful: three strikes on the same day still blacklist, while a node
468
+ * that failed once three days ago starts from 0 again. The remainder is
469
+ * rounded *up*, so a fresh single strike is not silently discounted to 0
470
+ * within minutes by the rounding step.
471
+ */
472
+ _decayedStrikeCount(stats, nowMs = Date.now()) {
473
+ if (stats.failureCount <= 0)
474
+ return 0;
475
+ if (!stats.lastFailureAtMs)
476
+ return stats.failureCount;
477
+ const decayMs = this.config.healthCheck.strikeDecayMs;
478
+ const age = Math.max(0, nowMs - stats.lastFailureAtMs);
479
+ if (age >= decayMs)
480
+ return 0;
481
+ const remainingFactor = 1 - age / decayMs;
482
+ return Math.max(0, Math.ceil(stats.failureCount * remainingFactor));
483
+ }
484
+ /**
485
+ * Record a successful health-probe result: set status/latency and let the
486
+ * strike count decay.
380
487
  *
381
- * A single successful probe must not erase strikes that came from the live
382
- * transport. Require `liveFailureSuccessStreak` consecutive successes before
383
- * clearing those, so a genuinely flapping node keeps accumulating toward
384
- * BLACKLIST_THRESHOLD while a recovered node is still re-admitted (status is
385
- * set either way). Strikes whose last source was a health check clear on the
386
- * first success, preserving normal probe recovery.
488
+ * This deliberately no longer *resets* strikes: the old
489
+ * liveFailureSuccessStreak rule cleared a live-transport strike after two
490
+ * successful probes, and with a 4h probe interval that forgave a
491
+ * once-a-day flapper twice a day — every failure logged "attempt 1/3" and
492
+ * the node was never blacklisted. Strikes now age out on their own
493
+ * (`strikeDecayMs`), so probe success can re-admit a node's *status*
494
+ * without erasing the evidence that it drops connections.
387
495
  *
388
- * @returns {{ recovered: boolean, failureCount: number }}
496
+ * @returns {{ recovered: boolean, failureCount: number }} whether every
497
+ * strike has aged out, and the remaining count.
389
498
  */
390
499
  _recordHealthCheckSuccess(nodeUrl, status, latencyMs, chainId) {
391
500
  const stats = this.nodeStats.get(nodeUrl);
392
501
  if (!stats)
393
502
  return { recovered: false, failureCount: 0 };
394
- const priorSource = stats.lastFailureSource;
395
- const liveFailure = !!priorSource && priorSource !== 'health-check';
396
- stats.healthCheckSuccessStreak = (stats.healthCheckSuccessStreak || 0) + 1;
397
- const successStreak = this.config.healthCheck.liveFailureSuccessStreak;
398
- const recovered = !liveFailure || stats.healthCheckSuccessStreak >= successStreak;
399
503
  stats.status = status;
400
504
  stats.latencyMs = latencyMs;
401
505
  stats.lastCheckTime = nowIso();
402
506
  stats.lastErrorMessage = null;
403
507
  stats.chainId = chainId;
404
- if (recovered) {
405
- stats.failureCount = 0;
406
- stats.lastFailureSource = null;
407
- stats.healthCheckSuccessStreak = 0;
408
- this.failureLedger.reset(nodeUrl);
409
- }
410
- else {
411
- this.logger.debug(`${nodeUrl.substring(0, 40)}... ${status} but retaining ${stats.failureCount} live-transport strike(s) ` +
412
- `(success streak ${stats.healthCheckSuccessStreak}/${successStreak})`);
508
+ const remaining = this._decayedStrikeCount(stats);
509
+ const recovered = remaining <= 0;
510
+ if (stats.failureCount > 0) {
511
+ if (remaining <= 0) {
512
+ stats.failureCount = 0;
513
+ stats.lastFailureAtMs = null;
514
+ this.failureLedger.reset(nodeUrl);
515
+ }
516
+ else {
517
+ stats.failureCount = remaining;
518
+ // Keep the ledger and the visible count in step, otherwise the
519
+ // next failure would blacklist on stale evidence.
520
+ this.failureLedger.seed(nodeUrl, { failureCount: remaining, blacklistedUntil: 0 });
521
+ this.logger.debug(`${nodeUrl.substring(0, 40)}... ${status} but retaining ${remaining} decaying strike(s)`);
522
+ }
413
523
  }
414
524
  return { recovered, failureCount: stats.failureCount };
415
525
  }
@@ -567,13 +677,40 @@ class NodeManager {
567
677
  });
568
678
  }
569
679
  /**
570
- * Get list of healthy nodes sorted by latency (best first)
680
+ * Flap-scoring options built from this manager's configuration.
681
+ * Single place that maps config -> the pure ranking maths, so the ranking,
682
+ * the logged counters and the persisted health cache always agree.
683
+ * @private
684
+ */
685
+ _flapOptions(nowMs = Date.now()) {
686
+ return {
687
+ nowMs,
688
+ windowMs: NODE_MANAGEMENT.FLAP_WINDOW_MS,
689
+ halfLifeMs: this.config.healthCheck.flapHalfLifeMs,
690
+ penaltyMsPerDrop: this.config.healthCheck.flapPenaltyMsPerDrop,
691
+ maxPenaltyMs: this.config.healthCheck.flapPenaltyMaxMs,
692
+ };
693
+ }
694
+ /**
695
+ * Get list of healthy nodes, best first.
696
+ *
697
+ * Ranking is `measured latency + decayed connection-churn penalty`, so a
698
+ * 50ms node that drops its socket daily loses to a stable 350ms node. See
699
+ * modules/node_flap_score.ts for the penalty maths.
571
700
  * @returns {string[]} Array of node URLs
572
701
  */
573
702
  getHealthyNodes() {
574
703
  const preferredNode = this.config.selection.preferredNode;
575
- const healthy = Array.from(this.nodeStats.values())
576
- .filter((stat) => stat.status === 'healthy' || stat.status === 'slow')
704
+ const flapOptions = this._flapOptions();
705
+ const healthyStats = Array.from(this.nodeStats.values())
706
+ .filter((stat) => stat.status === 'healthy' || stat.status === 'slow');
707
+ // Precompute one rank per node: doing it inside the comparator would
708
+ // prune and re-sort every node's drop history O(n log n) times.
709
+ const ranks = new Map();
710
+ for (const stat of healthyStats) {
711
+ ranks.set(stat, effectiveLatencyMs(stat.latencyMs, stat.dropTimes, flapOptions));
712
+ }
713
+ const healthy = healthyStats
577
714
  .sort((a, b) => {
578
715
  if (preferredNode) {
579
716
  const aPreferred = a.url === preferredNode && a.status === 'healthy';
@@ -581,13 +718,36 @@ class NodeManager {
581
718
  if (aPreferred !== bPreferred)
582
719
  return aPreferred ? -1 : 1;
583
720
  }
584
- return compareNodeHealth(a, b);
721
+ if (a.status !== b.status)
722
+ return a.status === 'healthy' ? -1 : 1;
723
+ // Compare without subtracting: two unknown latencies are both
724
+ // Infinity, and Infinity - Infinity is NaN (an unstable sort
725
+ // comparator).
726
+ const aRank = ranks.get(a) ?? Infinity;
727
+ const bRank = ranks.get(b) ?? Infinity;
728
+ if (aRank === bRank)
729
+ return 0;
730
+ return aRank < bRank ? -1 : 1;
585
731
  })
586
732
  .map((stat) => stat.url);
587
733
  return healthy;
588
734
  }
589
735
  /**
590
- * Get the single best (lowest latency) healthy node
736
+ * Connection-churn counters for one node (drops/h, decayed score, penalty).
737
+ * Used for logging and for the shared health cache.
738
+ * @param {string} nodeUrl - Node URL
739
+ * @returns {Object|null} Churn counters or null for an unknown node
740
+ */
741
+ getNodeChurn(nodeUrl, nowMs = Date.now()) {
742
+ const stats = this.nodeStats.get(nodeUrl);
743
+ if (!stats)
744
+ return null;
745
+ const flapOptions = this._flapOptions(nowMs);
746
+ const dropTimes = pruneDropTimes(stats.dropTimes, nowMs, flapOptions.windowMs);
747
+ return { dropTimes, ...dropCounters(dropTimes, flapOptions) };
748
+ }
749
+ /**
750
+ * Get the single best-ranked healthy node (latency + churn penalty)
591
751
  * @returns {string|null} Best node URL or null if none available
592
752
  */
593
753
  getBestNode() {
@@ -621,41 +781,55 @@ class NodeManager {
621
781
  * failure ledger — the same rules the credential daemon applies.
622
782
  * @param {string} nodeUrl - Node URL that failed
623
783
  * @param {string} [errorMessage] - Optional error description
624
- * @param {string} [source] - Failure source ('health-check' or 'broadcast'); controls log level
784
+ * @param {string} [source] - Failure source ('health-check', 'broadcast',
785
+ * 'connection', 'keep-alive'); controls log level and churn accounting
786
+ * @param {number} [nowMs] - Timestamp for the strike/drop (defaults to now;
787
+ * injectable so tests do not need to sleep past the report cooldown)
625
788
  */
626
- reportNodeFailure(nodeUrl, errorMessage, source) {
789
+ reportNodeFailure(nodeUrl, errorMessage, source, nowMs = Date.now()) {
627
790
  const stats = this.nodeStats.get(nodeUrl);
628
791
  if (!stats) {
629
792
  this.logger.warn(`Node ${nodeUrl} not in configured list, cannot report failure`);
630
793
  return;
631
794
  }
632
795
  const isHealthCheck = source === 'health-check';
633
- const result = this.failureLedger.recordFailure(nodeUrl);
796
+ const result = this.failureLedger.recordFailure(nodeUrl, nowMs);
634
797
  stats.failureCount = result.failureCount;
635
798
  stats.lastCheckTime = nowIso();
636
799
  if (errorMessage)
637
800
  stats.lastErrorMessage = errorMessage;
638
- // Remember the source so a later health-check success knows whether the
639
- // strike came from the live transport (retain) or a probe (clearable).
640
- // Recorded even when the ledger rate-limits this report.
641
- stats.lastFailureSource = source || 'unknown';
642
- stats.healthCheckSuccessStreak = 0;
801
+ const counted = result.outcome === 'counted' || result.outcome === 'blacklisted';
802
+ // A counted strike refreshes the decay clock. Recording it even for
803
+ // probe failures keeps `failureCount` and its age consistent.
804
+ if (counted) {
805
+ stats.lastFailureAtMs = nowMs;
806
+ }
807
+ // Only live-operation failures feed the churn score used by selection
808
+ // (see CHURN_SOURCES). Rate-limited reports restate the same event, so
809
+ // they are excluded along with blacklisted-node repeats.
810
+ if (counted && source && CHURN_SOURCES.has(source)) {
811
+ stats.dropTimes = recordDrop(stats.dropTimes, nowMs, this._flapOptions(nowMs).windowMs);
812
+ }
643
813
  if (result.outcome === 'rate-limited' || result.outcome === 'skipped-blacklisted') {
644
814
  return;
645
815
  }
646
816
  if (result.outcome === 'blacklisted') {
647
817
  stats.status = 'blacklisted';
648
- stats.blacklistedAt = Date.now();
818
+ stats.blacklistedAt = nowMs;
649
819
  this.saveBlacklistState();
650
820
  this.saveHealthCache();
651
821
  if (this._shouldLogBlacklistWarning(nodeUrl, errorMessage || '')) {
652
- this.logger.warn(`✗ ${nodeUrl.substring(0, 40)}... BLACKLISTED after ${result.failureCount} failures (${errorMessage || 'unknown'})`);
822
+ const churn = this.getNodeChurn(nodeUrl);
823
+ this.logger.warn(`✗ ${nodeUrl.substring(0, 40)}... BLACKLISTED after ${result.failureCount} failures (${errorMessage || 'unknown'}) ` +
824
+ `[churn ${churn ? `${churn.drops24h} drop(s)/24h, +${churn.penaltyMs}ms` : 'n/a'}]`);
653
825
  }
654
826
  }
655
827
  else {
656
828
  stats.status = 'failed';
657
829
  const log = isHealthCheck ? this.logger.debug.bind(this.logger) : this.logger.warn.bind(this.logger);
658
- log(`⚠ ${nodeUrl.substring(0, 40)}... FAILED attempt ${result.failureCount}/${result.threshold} (${errorMessage || 'unknown'})`);
830
+ const churn = this.getNodeChurn(nodeUrl);
831
+ log(`⚠ ${nodeUrl.substring(0, 40)}... FAILED attempt ${result.failureCount}/${result.threshold} (${errorMessage || 'unknown'})` +
832
+ (churn && churn.drops24h > 0 ? ` [churn ${churn.drops24h} drop(s)/24h, +${churn.penaltyMs}ms]` : ''));
659
833
  }
660
834
  }
661
835
  /**
@@ -668,6 +842,7 @@ class NodeManager {
668
842
  stats.status = 'blacklisted';
669
843
  stats.failureCount = this.config.healthCheck.blacklistThreshold;
670
844
  stats.blacklistedAt = Date.now();
845
+ stats.lastFailureAtMs = stats.blacklistedAt;
671
846
  this.failureLedger.seed(nodeUrl, {
672
847
  failureCount: stats.failureCount,
673
848
  blacklistedUntil: stats.blacklistedAt + this.BLACKLIST_COOLDOWN_MS,
@@ -689,8 +864,10 @@ class NodeManager {
689
864
  stats.latencyMs = null;
690
865
  stats.lastErrorMessage = null;
691
866
  stats.blacklistedAt = null;
692
- stats.lastFailureSource = null;
693
- stats.healthCheckSuccessStreak = 0;
867
+ stats.lastFailureAtMs = null;
868
+ // An explicit reset is the operator saying "give this node a clean
869
+ // slate": churn history goes with the strikes.
870
+ stats.dropTimes = [];
694
871
  this.failureLedger.reset(nodeUrl);
695
872
  this._clearBlacklistWarningCooldown(nodeUrl);
696
873
  this.saveBlacklistState();
@@ -709,8 +886,8 @@ class NodeManager {
709
886
  stats.lastErrorMessage = null;
710
887
  stats.chainId = null;
711
888
  stats.blacklistedAt = null;
712
- stats.lastFailureSource = null;
713
- stats.healthCheckSuccessStreak = 0;
889
+ stats.lastFailureAtMs = null;
890
+ stats.dropTimes = [];
714
891
  }
715
892
  this.failureLedger.clear();
716
893
  this._clearBlacklistWarningCooldown();
@@ -723,14 +900,21 @@ class NodeManager {
723
900
  * @returns {Array<Object>} Array of node stats
724
901
  */
725
902
  getStats() {
726
- return Array.from(this.nodeStats.values()).map((stat) => ({
727
- url: stat.url,
728
- status: stat.status,
729
- latencyMs: stat.latencyMs,
730
- failureCount: stat.failureCount,
731
- lastCheckTime: stat.lastCheckTime,
732
- lastErrorMessage: stat.lastErrorMessage
733
- }));
903
+ const nowMs = Date.now();
904
+ return Array.from(this.nodeStats.values()).map((stat) => {
905
+ const churn = this.getNodeChurn(stat.url, nowMs);
906
+ return {
907
+ url: stat.url,
908
+ status: stat.status,
909
+ latencyMs: stat.latencyMs,
910
+ failureCount: stat.failureCount,
911
+ lastCheckTime: stat.lastCheckTime,
912
+ lastErrorMessage: stat.lastErrorMessage,
913
+ drops24h: churn ? churn.drops24h : 0,
914
+ dropsPerHour: churn ? churn.dropsPerHour : 0,
915
+ flapPenaltyMs: churn ? churn.penaltyMs : 0
916
+ };
917
+ });
734
918
  }
735
919
  /**
736
920
  * Get summary statistics
@@ -755,11 +939,17 @@ class NodeManager {
755
939
  const avgLatency = statsWithLatency.length > 0
756
940
  ? Math.round(statsWithLatency.reduce((sum, s) => sum + s.latencyMs, 0) / statsWithLatency.length)
757
941
  : null;
942
+ const nowMs = Date.now();
943
+ const totalDrops24h = stats.reduce((sum, s) => {
944
+ const churn = this.getNodeChurn(s.url, nowMs);
945
+ return sum + (churn ? churn.drops24h : 0);
946
+ }, 0);
758
947
  return {
759
948
  monitoring: this.monitoringActive,
760
949
  counts,
761
950
  bestNode,
762
- avgLatency
951
+ avgLatency,
952
+ totalDrops24h
763
953
  };
764
954
  }
765
955
  }
@@ -90,7 +90,8 @@ import { ORDER_TYPES, ORDER_STATES, PIPELINE_TIMING, TIMING, FEE_PARAMETERS, GRI
90
90
  import { resolveAccountRef } from './utils/system.js';
91
91
  import { resolveSpreadOrderSide, parseChainOrder, applyChainSizeToGridOrder, convertToSpreadPlaceholder, resolveEdgeCount } from './utils/order.js';
92
92
  import { parseSlotIndex } from './utils/slot.js';
93
- import { isSlotInRail, floatToBlockchainInt } from './utils/math.js';
93
+ import { readSlotLifecycle } from './lifecycle.js';
94
+ import { isSlotInRail, resolveGapSlots, floatToBlockchainInt } from './utils/math.js';
94
95
  import * as Format from './format.js';
95
96
  import * as fundRegistry from '../fund_registry.js';
96
97
  import * as chainOrders from '../chain_orders.js';
@@ -372,7 +373,11 @@ class Accountant {
372
373
  // * price >= startPrice → SELL side (higher prices are asks)
373
374
  const genesis = mgr._genesis;
374
375
  const boundaryIdx = mgr.boundaryIdx;
375
- const gapSlots = genesis?.gapSlots ?? mgr._gapSlots ?? 0;
376
+ // Never fall back to gap 0: a 0-wide band classifies the slot just
377
+ // above the boundary as a SELL rail slot and would book a SPREAD
378
+ // placeholder's size on the wrong side. resolveGapSlots keeps the
379
+ // config-derived geometry as the fallback.
380
+ const gapSlots = genesis?.gapSlots ?? resolveGapSlots(mgr);
376
381
  let spreadSide = null;
377
382
  if (order.type === ORDER_TYPES.SPREAD && genesis && boundaryIdx != null && Number.isFinite(Number(boundaryIdx))) {
378
383
  if (isSlotInRail(boundaryIdx, gapSlots, ORDER_TYPES.BUY, order))
@@ -931,11 +936,12 @@ class Accountant {
931
936
  mgr.logger?.log?.(`[RECOVERY] Skipping recalibration for ${gridOrder.id}: live order ${gridOrder.orderId} is unparseable (unknown, not absent)`, 'debug');
932
937
  continue;
933
938
  }
934
- const commitAge = Date.now() - (mgr._committedOrderIdsBuiltAt || 0);
935
- const recentCommit = mgr._committedOrderIds?.has?.(gridOrder.orderId) && commitAge < TIMING.SYNC_LOCK_TIMEOUT_MS;
936
- const assignedAt = mgr._orderIdAssignedAt?.get?.(gridOrder.orderId) || 0;
937
- const recentAssign = assignedAt > 0 && (Date.now() - assignedAt) < TIMING.SYNC_LOCK_TIMEOUT_MS;
938
- if (recentCommit || recentAssign)
939
+ // Single in-flight guard (modules/order/lifecycle.ts).
940
+ // This site used to OR together `_committedOrderIds` and
941
+ // `_orderIdAssignedAt` while the startup reconcile OR'd
942
+ // neither-or-one; same order, different answer. Read through the
943
+ // SLOT so the persisted create-uncertain marker participates.
944
+ if (readSlotLifecycle(mgr, gridOrder).isInFlight())
939
945
  continue;
940
946
  result.attempted++;
941
947
  const spreadOrder = convertToSpreadPlaceholder(gridOrder);
@@ -1642,6 +1648,20 @@ class Accountant {
1642
1648
  const assetBPrecision = mgr.assets?.assetB?.precision;
1643
1649
  if (assetAPrecision === undefined || assetBPrecision === undefined)
1644
1650
  return false;
1651
+ // Idempotency with sync pass 1: an open-orders pass may already have
1652
+ // resized the slot from the same post-fill chain snapshot (and
1653
+ // released that commitment into free). The fill transition must not
1654
+ // shrink/release the same base units a second time (3.6604 -> 3.3046
1655
+ // -> 2.9488). The trade's pays/receives are still applied here — pass
1656
+ // 1 never books them. A limit-order fill always PAYS the order's base
1657
+ // asset (assetA for a SELL, assetB for a BUY); the order's `size` and
1658
+ // the pass-1 delta are in that same base unit. `receives` is the
1659
+ // quote side and must not be used for the marker.
1660
+ const filledBaseInt = Number(pays.amount);
1661
+ // The marker is consumed LATER, just before the adjustments are
1662
+ // applied: a duplicate fill key or a failed dedup-key persist must not
1663
+ // eat marker units a following split fill of the same order needs.
1664
+ let appliedInt = 0;
1645
1665
  // Derive all numeric effects before recording the fill key.
1646
1666
  // This keeps retries safe if a later computation unexpectedly fails.
1647
1667
  const balanceAdjustments = [];
@@ -1669,6 +1689,13 @@ class Accountant {
1669
1689
  if (btsRefundAdjustment) {
1670
1690
  balanceAdjustments.push(btsRefundAdjustment);
1671
1691
  }
1692
+ // IMPORTANT: do NOT clear `balanceAdjustments` when pass 1 already
1693
+ // resized the slot. Pass 1's updateOptimisticFreeBalance only RELEASES
1694
+ // the commitment into free; it never applies the trade's pays/receives.
1695
+ // The duplicate the marker protects against is the second SIZE release
1696
+ // in the fill transition, which is handled by __chainSizeAppliedInt
1697
+ // (subtract only the remainder). Zeroing this list dropped the actual
1698
+ // trade until the post-batch reanchor happened to repair it.
1672
1699
  let processedAt = null;
1673
1700
  const tracker = this.manager.processedFillTracker;
1674
1701
  if (fillKey) {
@@ -1693,6 +1720,23 @@ class Accountant {
1693
1720
  return false;
1694
1721
  }
1695
1722
  }
1723
+ // Consume the pass-1 marker only now that this fill is not a
1724
+ // duplicate and its dedup key is queued. Partial consumption: pass 1
1725
+ // may have absorbed only part of this fill (A+B observed, fill A
1726
+ // drained now); the transition subtracts only the remainder.
1727
+ const consumeApplied = mgr.consumeChainSizeAppliedFillAmount;
1728
+ if (Number.isFinite(filledBaseInt) && typeof consumeApplied === 'function') {
1729
+ appliedInt = Math.max(0, Math.min(Math.round(Number(consumeApplied.call(mgr, String(fillOp.order_id), filledBaseInt)) || 0), Math.round(filledBaseInt)));
1730
+ }
1731
+ if (appliedInt > 0) {
1732
+ fillOp.__chainSizeAlreadyApplied = true;
1733
+ fillOp.__chainSizeAppliedInt = appliedInt;
1734
+ mgr?.logger?.log?.(`[FILL-DEDUP] Fill ${fillKey || fillOp.order_id}: ${appliedInt}/${Math.round(filledBaseInt)} base units already absorbed by sync pass 1; the transition will not shrink them again`, 'debug');
1735
+ }
1736
+ else {
1737
+ delete fillOp.__chainSizeAlreadyApplied;
1738
+ delete fillOp.__chainSizeAppliedInt;
1739
+ }
1696
1740
  await this._applyBalanceAdjustments(balanceAdjustments);
1697
1741
  // Flush the queued dedup key now that balance adjustments succeeded.
1698
1742
  // For IMMEDIATE mode, this persists durably. For BATCHED mode, the
@@ -1710,6 +1754,11 @@ class Accountant {
1710
1754
  await this.adjustTotalBalance(adj.orderType, -adj.delta, 'rollback-' + adj.operation);
1711
1755
  }
1712
1756
  processedFillStore.discard(fillKey, processedAt || Date.now());
1757
+ // The marker was consumed for this attempt; the rolled-back
1758
+ // fill must be able to skip the pass-1 size release on retry.
1759
+ if (appliedInt > 0 && typeof mgr.noteChainSizeAppliedFill === 'function') {
1760
+ mgr.noteChainSizeAppliedFill(String(fillOp.order_id), appliedInt);
1761
+ }
1713
1762
  throw err;
1714
1763
  }
1715
1764
  }