dexbot 1.7.0 → 1.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +3 -3
  3. package/analysis/README.md +33 -13
  4. package/claw/package.json +1 -1
  5. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  6. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  7. package/dist/analysis/account_resolver.js +5 -6
  8. package/dist/analysis/analyze_liquidity.js +15 -4
  9. package/dist/analysis/bot_key_utils.d.ts +44 -1
  10. package/dist/analysis/bot_key_utils.js +76 -7
  11. package/dist/analysis/fills_cache.js +2 -1
  12. package/dist/analysis/grid_correction_check.d.ts +1 -19
  13. package/dist/analysis/grid_correction_check.js +12 -30
  14. package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
  15. package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
  16. package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
  17. package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
  18. package/dist/analysis/liquidity/liquidity_view.js +11 -0
  19. package/dist/analysis/resolve_bot_accounts.js +7 -6
  20. package/dist/analysis/trade_profitability.js +2 -1
  21. package/dist/credential-daemon.js +13 -2
  22. package/dist/dexbot.js +4 -4
  23. package/dist/modules/account_bots.d.ts +12 -1
  24. package/dist/modules/account_bots.js +7 -5
  25. package/dist/modules/account_orders.d.ts +12 -0
  26. package/dist/modules/account_orders.js +30 -0
  27. package/dist/modules/bitshares-native/transport.js +24 -6
  28. package/dist/modules/bitshares_client.d.ts +24 -6
  29. package/dist/modules/bitshares_client.js +44 -7
  30. package/dist/modules/bot_settings.d.ts +2 -0
  31. package/dist/modules/constants.d.ts +28 -3
  32. package/dist/modules/constants.js +79 -10
  33. package/dist/modules/dexbot_class.d.ts +23 -6
  34. package/dist/modules/dexbot_class.js +44 -0
  35. package/dist/modules/dexbot_cow_runtime.d.ts +128 -8
  36. package/dist/modules/dexbot_cow_runtime.js +688 -188
  37. package/dist/modules/dexbot_fill_runtime.js +124 -35
  38. package/dist/modules/dexbot_maintenance_runtime.d.ts +1 -1
  39. package/dist/modules/dexbot_maintenance_runtime.js +259 -79
  40. package/dist/modules/dexbot_startup_runtime.js +32 -20
  41. package/dist/modules/dexbot_state_recovery.d.ts +33 -1
  42. package/dist/modules/dexbot_state_recovery.js +167 -5
  43. package/dist/modules/node_connect_policy.d.ts +8 -1
  44. package/dist/modules/node_connect_policy.js +18 -2
  45. package/dist/modules/node_flap_score.d.ts +87 -0
  46. package/dist/modules/node_flap_score.js +124 -0
  47. package/dist/modules/node_health_cache.d.ts +40 -3
  48. package/dist/modules/node_health_cache.js +66 -20
  49. package/dist/modules/node_manager.d.ts +101 -19
  50. package/dist/modules/node_manager.js +255 -65
  51. package/dist/modules/order/accounting.d.ts +9 -4
  52. package/dist/modules/order/accounting.js +131 -19
  53. package/dist/modules/order/async_lock.d.ts +7 -1
  54. package/dist/modules/order/async_lock.js +7 -1
  55. package/dist/modules/order/export.js +25 -18
  56. package/dist/modules/order/fill_ledger.d.ts +99 -0
  57. package/dist/modules/order/fill_ledger.js +103 -0
  58. package/dist/modules/order/format.d.ts +24 -10
  59. package/dist/modules/order/format.js +30 -10
  60. package/dist/modules/order/grid.d.ts +75 -18
  61. package/dist/modules/order/grid.js +289 -72
  62. package/dist/modules/order/grid_reconcile.d.ts +5 -1
  63. package/dist/modules/order/grid_reconcile.js +333 -210
  64. package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
  65. package/dist/modules/order/grid_reconcile_internal.js +223 -91
  66. package/dist/modules/order/index.d.ts +18 -5
  67. package/dist/modules/order/index.js +5 -5
  68. package/dist/modules/order/lifecycle.d.ts +353 -0
  69. package/dist/modules/order/lifecycle.js +1109 -0
  70. package/dist/modules/order/manager.d.ts +277 -20
  71. package/dist/modules/order/manager.js +680 -64
  72. package/dist/modules/order/placement_lock.d.ts +102 -0
  73. package/dist/modules/order/placement_lock.js +222 -0
  74. package/dist/modules/order/strategy.js +13 -1
  75. package/dist/modules/order/sync_engine.d.ts +74 -114
  76. package/dist/modules/order/sync_engine.js +254 -198
  77. package/dist/modules/order/utils/math.d.ts +29 -4
  78. package/dist/modules/order/utils/math.js +61 -4
  79. package/dist/modules/order/utils/order.d.ts +106 -37
  80. package/dist/modules/order/utils/order.js +256 -81
  81. package/dist/modules/order/utils/system.d.ts +99 -6
  82. package/dist/modules/order/utils/system.js +249 -35
  83. package/dist/modules/order/utils/validate.d.ts +7 -1
  84. package/dist/modules/order/utils/validate.js +48 -2
  85. package/dist/modules/outcome.d.ts +54 -0
  86. package/dist/modules/outcome.js +51 -0
  87. package/dist/modules/settings_merge.d.ts +4 -0
  88. package/dist/modules/settings_merge.js +9 -0
  89. package/dist/modules/timing_policy.d.ts +82 -0
  90. package/dist/modules/timing_policy.js +244 -0
  91. package/dist/modules/types.d.ts +149 -22
  92. package/dist/scripts/analyze-credit.js +54 -17
  93. package/dist/scripts/analyze-orders.d.ts +4 -2
  94. package/dist/scripts/analyze-orders.js +113 -34
  95. package/dist/scripts/chart_targets.js +11 -15
  96. package/dist/scripts/test-credit-renewal.js +12 -5
  97. package/docs/COW_INVARIANTS.md +71 -8
  98. package/docs/DEXBOT_COMPARISON.md +6 -6
  99. package/docs/EVOLUTION.md +10 -8
  100. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  101. package/docs/GRID_PRICE_INVARIANT.md +1 -1
  102. package/docs/GRID_RECONCILE.md +18 -13
  103. package/docs/LOGGING.md +25 -1
  104. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +183 -26
  105. package/docs/README.md +23 -2
  106. package/docs/WORKFLOW.md +9 -3
  107. package/docs/architecture.md +68 -13
  108. package/docs/developer_guide.md +25 -8
  109. package/modules/README.md +6 -0
  110. package/package.json +3 -3
  111. package/scripts/README.md +16 -0
  112. package/scripts/pdev.sh +14 -1
  113. package/scripts/pmain.sh +23 -8
  114. package/scripts/ptest.sh +7 -0
  115. package/tests/README.md +30 -1
@@ -6,9 +6,9 @@ import { readOpenOrdersGuarded } from './chain_orders.js';
6
6
  import { ORDER_TYPES, TIMING } from './constants.js';
7
7
  import * as Format from './order/format.js';
8
8
  import * as grid from './order/grid.js';
9
- import { convertToSpreadPlaceholder, parseChainOrder, isNonBlockingUnmatchedOrder } from './order/utils/order.js';
10
- import { restoreGapEvacStreaks, applyPersistedPendingCrawls, resetLastFillPivot } from './order/utils/system.js';
11
- import { blockchainToFloat, calculateGapSlots, validatePersistedBoundary, isTransientInBandRejection } from './order/utils/math.js';
9
+ import { convertToSpreadPlaceholder, parseChainOrder, isNonBlockingUnmatchedOrder, detectGapEvacuationCandidates, updateGapEvacuationStreaks } from './order/utils/order.js';
10
+ import { restoreGapEvacStreaks, applyPersistedPendingCrawls, resetLastFillPivot, hasUnsettledFills } from './order/utils/system.js';
11
+ import { blockchainToFloat, calculateGapSlots, resolveGapSlots, validatePersistedBoundary, isTransientInBandRejection } from './order/utils/math.js';
12
12
  import { ON_MISSING_GENESIS, isMissingGenesisError } from './order/genesis_policy.js';
13
13
  import { hasExecutableActions } from './order/utils/validate.js';
14
14
  import { getErrorMessage } from './utils/errors.js';
@@ -68,6 +68,7 @@ async function triggerStateRecoverySync(bot, reason = 'state recovery sync') {
68
68
  return;
69
69
  }
70
70
  bot._recoverySyncInFlight++;
71
+ bot._recoverySyncInFlightStartedAt = Date.now();
71
72
  try {
72
73
  bot.manager.logger.log(`Triggering state recovery sync (${reason})...`, 'info');
73
74
  await bot.manager.fetchAccountTotals(bot.accountId);
@@ -96,7 +97,10 @@ async function triggerStateRecoverySync(bot, reason = 'state recovery sync') {
96
97
  bot.manager.logger.log(`[RECOVERY] State recovery sync (${reason}) failed: ${getErrorMessage(err)}. Preserving current state; the next sync cycle will retry.`, 'error');
97
98
  }
98
99
  finally {
99
- bot._recoverySyncInFlight--;
100
+ bot._recoverySyncInFlight = Math.max(0, bot._recoverySyncInFlight - 1);
101
+ if (bot._recoverySyncInFlight === 0) {
102
+ bot._recoverySyncInFlightStartedAt = 0;
103
+ }
100
104
  }
101
105
  }
102
106
  /**
@@ -484,6 +488,142 @@ async function recoverFromPersistedGrid(bot) {
484
488
  return { success: false, reason: getErrorMessage(err) };
485
489
  }
486
490
  }
491
+ /**
492
+ * Rate limit for gap-evac repairs (one fill-less re-plan per bot per minute).
493
+ * The post-commit hook and the maintenance sweep can both observe the same
494
+ * stranding; without a floor a plan that fails to resolve it (guard, funds,
495
+ * crossing) would re-plan on every commit and burn broadcasts.
496
+ */
497
+ const GAP_EVAC_REPAIR_COOLDOWN_MS = 60 * TIMING.MILLISECONDS_PER_SECOND;
498
+ const _lastGapEvacRepairAt = new WeakMap();
499
+ /**
500
+ * Detect live orders resting INSIDE the committed gap band and schedule the
501
+ * repair (fill-less COW re-plan + cancel-only teeth as last resort).
502
+ *
503
+ * Why this exists: a boundary commit that moves the boundary away from a live
504
+ * order strands it inside the spread band. The plan normally evacuates it with
505
+ * a rotation in the SAME batch, but the LAST-FILL-GUARD re-proves an evacuation
506
+ * against the PRE-COMMIT geometry and refuses it as stale when the plan's own
507
+ * crawl is what created the stranding (fail-closed by design — see the
508
+ * 2026-10-30 bid-edge incident). The batch then commits the new boundary with
509
+ * the stranded order still live, and because plans are fill-driven the state
510
+ * never heals while the market is quiet: the 4-slot spread band keeps two live
511
+ * buys and the real spread collapses (observed: 1.21% against a 2.0% target).
512
+ *
513
+ * Re-planning AFTER the commit is safe where the in-batch rotation is not: the
514
+ * committed geometry is now the LIVE geometry, so the evacuation stamp passes
515
+ * `isEvacuationStampStillValid` and the rotation executes as one price update.
516
+ *
517
+ * Cheap when healthy: one `detectGapEvacuationCandidates` scan (O(grid)) and an
518
+ * early return. Callers run it on commit (writer of the stranding) and on the
519
+ * maintenance tick (covers restarts and commits from paths without a bot
520
+ * reference).
521
+ *
522
+ * @param {import('./dexbot_class.js').DEXBot} bot
523
+ * @param {string} context - Label for logging/telemetry
524
+ * @returns `{ stranded, scheduled }` — stranded slot ids found, whether a
525
+ * re-plan was scheduled.
526
+ */
527
+ function scheduleGapEvacRepair(bot, context = 'gap-evac sweep') {
528
+ const manager = bot.manager;
529
+ if (!manager || !(manager.orders instanceof Map) || manager.orders.size === 0) {
530
+ return { stranded: [], scheduled: false };
531
+ }
532
+ try {
533
+ // UNKNOWN GEOMETRY IS NOT HEALTH. Without a committed boundary (or a
534
+ // known band width) there is no band to judge, so this sweep must not
535
+ // conclude "clean" — clearing the ledger or scheduling against a guessed
536
+ // band would be the same "absent reads as 0" trap that hid the stranding
537
+ // in the first place. Same fail-closed contract as
538
+ // isEvacuationStampStillValid.
539
+ const boundaryKnown = Number.isFinite(Format.toKnownNumber(manager.boundaryIdx));
540
+ const gapKnown = Number.isFinite(Format.toKnownNumber(manager._gapSlots))
541
+ || Number.isFinite(Format.toKnownNumber(manager._genesis?.gapSlots));
542
+ if (!boundaryKnown || !gapKnown)
543
+ return { stranded: [], scheduled: false };
544
+ const gapSlots = resolveGapSlots(manager);
545
+ const candidates = detectGapEvacuationCandidates(manager.orders, manager.boundaryIdx, gapSlots, { assets: manager.assets, genesis: manager._genesis });
546
+ const streaks = manager._gapEvacStreaks instanceof Map ? manager._gapEvacStreaks : null;
547
+ if (candidates.length === 0) {
548
+ // Healthy: drop resolved ledger entries so a future return to the
549
+ // band re-arms instead of inheriting a stale cycle count.
550
+ const resolved = streaks ? streaks.size : 0;
551
+ if (streaks && resolved > 0) {
552
+ updateGapEvacuationStreaks(streaks, []);
553
+ manager.logger?.log?.(`[GAP-EVAC] Spread band clean: cleared ${resolved} resolved in-band streak(s)`, 'debug');
554
+ }
555
+ return { stranded: [], scheduled: false };
556
+ }
557
+ const stranded = candidates.map((c) => `${c.id}(${c.type})`);
558
+ // Re-derive the streak ledger against the COMMITTED geometry (drops
559
+ // resolved ids) WITHOUT counting a new observation: reconcileGrid ticks
560
+ // this same ledger at plan-build time, and counting twice per cycle
561
+ // armed the cancel-only teeth in the very cycle whose rotation was about
562
+ // to evacuate the order. Observed live: the sweep's tick plus the plan's
563
+ // tick took a restored streak of 1 to 3 (cancel threshold), the teeth
564
+ // queued a cancel for both rotation sources, the pre-batch drain
565
+ // cancelled them, and the UPDATE batch was rejected pre-broadcast with
566
+ // "2 target(s) missing on chain" → structural resync + full grid reload
567
+ // instead of two in-place price updates.
568
+ const tick = streaks
569
+ ? updateGapEvacuationStreaks(streaks, candidates, null, { increment: false })
570
+ : { streaks: {}, ready: [] };
571
+ // The cancel-only teeth stay the LAST resort. `_processGapEvacuationTeeth`
572
+ // returns the number it actually queued (it re-verifies live geometry and
573
+ // honours queued-once), so report that — not the number of candidates it
574
+ // was offered.
575
+ const queueTeeth = (ready) => {
576
+ const teeth = manager._processGapEvacuationTeeth;
577
+ if (typeof teeth !== 'function' || ready.length === 0)
578
+ return 0;
579
+ const queued = Number(teeth.call(manager, ready));
580
+ return Number.isFinite(queued) && queued > 0 ? queued : 0;
581
+ };
582
+ // The repair is a broadcast: only claim to schedule it when the bot can
583
+ // actually run a fill-less rebalance (same precondition the scheduler
584
+ // itself checks), so the log never promises a repair that cannot run.
585
+ if (typeof manager.performSafeRebalance !== 'function'
586
+ || typeof bot.updateOrdersOnChainBatch !== 'function') {
587
+ // No plan cycle can run on this path, so this sweep is the ONLY
588
+ // ticker for the observation: advance the ledger before consulting it.
589
+ const ready = streaks ? updateGapEvacuationStreaks(streaks, candidates).ready : tick.ready;
590
+ const queued = queueTeeth(ready);
591
+ manager.logger?.log?.(`[GAP-EVAC] ${stranded.length} live order(s) stranded inside the spread band ` +
592
+ `(boundary ${manager.boundaryIdx}, gap ${gapSlots}): ${stranded.join(', ')} — ` +
593
+ `no fill-less re-plan available on this path; teeth queued=${queued}`,
594
+ // Warn only when this tick actually did something; a bot shape
595
+ // without a re-plan path must not warn on every tick.
596
+ queued > 0 ? 'warn' : 'debug');
597
+ return { stranded, scheduled: false };
598
+ }
599
+ const now = Date.now();
600
+ const lastAt = _lastGapEvacRepairAt.get(bot) ?? 0;
601
+ const cooling = lastAt > 0 && (now - lastAt) < GAP_EVAC_REPAIR_COOLDOWN_MS;
602
+ if (cooling) {
603
+ // The previous re-plan either is still in flight or did not clear
604
+ // the stranding: fall back to the cancel-only teeth so a repeatedly
605
+ // unfixable slot cannot sit in the band forever. Readiness comes from
606
+ // the ledger as the PLAN cycles left it — scheduled re-plans keep
607
+ // running through the cooldown and each ticks it once, so the sweep
608
+ // must not count the same observation again.
609
+ const queued = queueTeeth(tick.ready);
610
+ manager.logger?.log?.(`[GAP-EVAC] ${stranded.length} live order(s) inside the spread band (${stranded.join(', ')}) — ` +
611
+ `repair cooldown active (${Math.round((GAP_EVAC_REPAIR_COOLDOWN_MS - (now - lastAt)) / 1000)}s left); ` +
612
+ `teeth queued=${queued}`, queued > 0 ? 'warn' : 'debug');
613
+ return { stranded, scheduled: false };
614
+ }
615
+ _lastGapEvacRepairAt.set(bot, now);
616
+ manager.logger?.log?.(`[GAP-EVAC] ${stranded.length} live order(s) stranded inside the spread band ` +
617
+ `(boundary ${manager.boundaryIdx}, gap ${gapSlots}): ${stranded.join(', ')} — ` +
618
+ `scheduling a fill-less re-plan to evacuate (${context})`, 'warn');
619
+ _schedulePostRecoveryRebalance(bot, `gap-evac repair (${context})`);
620
+ return { stranded, scheduled: true };
621
+ }
622
+ catch (err) {
623
+ bot.manager?.logger?.log?.(`[GAP-EVAC] Stranding sweep failed (non-fatal): ${getErrorMessage(err)}`, 'debug');
624
+ return { stranded: [], scheduled: false };
625
+ }
626
+ }
487
627
  /**
488
628
  * Best-effort target-grid rebalance after a snapshot reload.
489
629
  *
@@ -583,6 +723,28 @@ function _schedulePostRecoveryRebalance(bot, context) {
583
723
  async function rejectCorruptedGridSnapshot(bot, context) {
584
724
  if (!bot.manager?.checkFundDriftAfterFills)
585
725
  return false;
726
+ // A drift measured while fills are queued/draining/deferred (or while a
727
+ // COW/recovery batch is in flight) is NOT snapshot corruption: the queued
728
+ // fills still have to move the grid/books. Deleting the snapshot here
729
+ // destroys a valid grid and the following recovery does nothing
730
+ // (2026-11-03: drift 6234.93639 == the two queued fills; another snapshot
731
+ // rejected as corrupted while deferred fills were pending). Gate the
732
+ // destructive decision on a settled pipeline.
733
+ const pipeline = hasUnsettledFills(bot);
734
+ const batchInFlight = Number(bot._batchInFlight) || 0;
735
+ const recoveryInFlight = Number(bot._recoverySyncInFlight) || 0;
736
+ const placementHeld = typeof bot.manager.isPlacementActive === 'function' && bot.manager.isPlacementActive() === true;
737
+ const deferredRebalance = Number(bot.manager._deferredRebalanceAt) || 0;
738
+ if (pipeline.unsettled || batchInFlight > 0 || recoveryInFlight > 0 || placementHeld || deferredRebalance > 0) {
739
+ const why = pipeline.reason
740
+ || (batchInFlight > 0 ? 'batch in flight'
741
+ : recoveryInFlight > 0 ? 'recovery sync in flight'
742
+ : placementHeld ? `placement region held by ${bot.manager.placementKind?.() || 'unknown'}`
743
+ : 'deferred rebalance pending');
744
+ bot._warn?.(`[SNAPSHOT-REJECT] Skipping corruption check (${context}): pipeline not settled — ` +
745
+ `${why}; the snapshot is preserved and re-checked once fills settle`);
746
+ return false;
747
+ }
586
748
  const driftCheck = bot.manager.checkFundDriftAfterFills();
587
749
  if (driftCheck.isValid)
588
750
  return false;
@@ -694,5 +856,5 @@ async function targetedOrderRepair(bot, orderIds) {
694
856
  return false;
695
857
  }
696
858
  }
697
- export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
859
+ export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, scheduleGapEvacRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
698
860
  //# sourceMappingURL=dexbot_state_recovery.js.map
@@ -41,7 +41,14 @@ type ConnectedNodePolicyAction = {
41
41
  * - `switch`: active node is blacklisted and healthy alternatives exist;
42
42
  * reconnect to the healthy set (deferred by the caller).
43
43
  * - `align`: active node is fine; keep the transport candidate list aligned
44
- * with the healthy set.
44
+ * with the healthy set, *including the node we are currently connected to*.
45
+ * The transport only keeps its socket on `connect()` when
46
+ * `nodeList.includes(activeNode)` (see bitshares-native/transport.ts), so
47
+ * dropping the active node from the candidate list forces a full
48
+ * disconnect/reconnect on the next `connectClient()` — even though the
49
+ * socket was healthy. Appending it as the last candidate is enough to keep
50
+ * the no-op path reachable and costs nothing: it is only used if every
51
+ * preferred node fails.
45
52
  * - `none`: nothing to do (health checks disabled, a reconnect is already in
46
53
  * progress, or there is no usable healthy set).
47
54
  *
@@ -5,7 +5,14 @@
5
5
  * - `switch`: active node is blacklisted and healthy alternatives exist;
6
6
  * reconnect to the healthy set (deferred by the caller).
7
7
  * - `align`: active node is fine; keep the transport candidate list aligned
8
- * with the healthy set.
8
+ * with the healthy set, *including the node we are currently connected to*.
9
+ * The transport only keeps its socket on `connect()` when
10
+ * `nodeList.includes(activeNode)` (see bitshares-native/transport.ts), so
11
+ * dropping the active node from the candidate list forces a full
12
+ * disconnect/reconnect on the next `connectClient()` — even though the
13
+ * socket was healthy. Appending it as the last candidate is enough to keep
14
+ * the no-op path reachable and costs nothing: it is only used if every
15
+ * preferred node fails.
9
16
  * - `none`: nothing to do (health checks disabled, a reconnect is already in
10
17
  * progress, or there is no usable healthy set).
11
18
  *
@@ -28,8 +35,17 @@ export function resolveConnectedNodeAction(input) {
28
35
  }
29
36
  const healthy = input.getHealthyNodes();
30
37
  if (healthy.length > 0) {
31
- return { action: 'align', nodes: healthy };
38
+ return { action: 'align', nodes: withActiveNodeLast(healthy, activeNode) };
32
39
  }
33
40
  return { action: 'none' };
34
41
  }
42
+ /**
43
+ * Healthy set with the active node appended when it is missing.
44
+ * Never reorders an existing candidate list.
45
+ */
46
+ function withActiveNodeLast(nodes, activeNode) {
47
+ if (!activeNode || nodes.includes(activeNode))
48
+ return nodes;
49
+ return [...nodes, activeNode];
50
+ }
35
51
  //# sourceMappingURL=node_connect_policy.js.map
@@ -0,0 +1,87 @@
1
+ /**
2
+ * modules/node_flap_score.ts - Time-decayed connection-churn (flapping) scoring
3
+ *
4
+ * Node selection used to rank nodes by raw RPC latency only, so the
5
+ * lowest-ping node won even when it dropped its socket every few hours. Live
6
+ * logs showed exactly that: three Cloudflare-fronted endpoints answered in
7
+ * 50-120ms and accounted for every abnormal close, while three
8
+ * 300-360ms endpoints never dropped once — and were never selected.
9
+ *
10
+ * This module turns transport drop *timestamps* into a ranking input:
11
+ *
12
+ * effectiveLatency = latencyMs + flapPenaltyMs
13
+ * flapScore = Σ 0.5 ^ (age_i / halfLife)
14
+ * flapPenaltyMs = min(flapScore * DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_PENALTY_MAX_MS)
15
+ *
16
+ * A node with no recent drops ranks on its ping alone. One undecayed drop costs
17
+ * DEFAULT_FLAP_PENALTY_MS_PER_DROP of effective latency and then fades with the
18
+ * half-life, so a node that flaps once and then behaves is re-admitted
19
+ * automatically, a node that keeps flapping stays at the back of the list, and
20
+ * nothing has to be "reset" by a lucky health probe (the old
21
+ * LIVE_FAILURE_HEALTH_SUCCESS_STREAK wipe-on-success counter).
22
+ *
23
+ * Everything here is pure: callers own persistence and logging. Drop history
24
+ * is bounded by the window (24h default) and capped by MAX_DROP_TIMES, so it
25
+ * is safe to persist in the shared health cache (see node_health_cache.ts).
26
+ */
27
+ /** Trailing window of drops that may influence selection. */
28
+ declare const DEFAULT_FLAP_WINDOW_MS: number;
29
+ /** Time for a drop's penalty weight to halve. */
30
+ declare const DEFAULT_FLAP_HALF_LIFE_MS: number;
31
+ /** Effective-latency cost of one *undecayed* drop. */
32
+ declare const DEFAULT_FLAP_PENALTY_MS_PER_DROP = 5000;
33
+ /** Ceiling for the decayed penalty, so a badly degraded node stays usable as a last resort. */
34
+ declare const DEFAULT_FLAP_PENALTY_MAX_MS: number;
35
+ /** Hard cap on retained drop timestamps per node (bounded persistence). */
36
+ declare const MAX_DROP_TIMES = 64;
37
+ /** Trailing window used for the reported drops/hour counters. */
38
+ declare const DROP_RATE_WINDOW_MS: number;
39
+ interface FlapScoreOptions {
40
+ nowMs?: number;
41
+ windowMs?: number;
42
+ halfLifeMs?: number;
43
+ penaltyMsPerDrop?: number;
44
+ maxPenaltyMs?: number;
45
+ }
46
+ interface DropCounters {
47
+ /** Drops inside the last hour (the literal "drops/h" signal). */
48
+ dropsLastHour: number;
49
+ /** Drops inside the trailing `windowMs` (24h by default). */
50
+ drops24h: number;
51
+ /** Averaged drops per hour across the window. */
52
+ dropsPerHour: number;
53
+ /** Decayed drop weight used for ranking. */
54
+ flapScore: number;
55
+ /** Effective latency penalty in ms. */
56
+ penaltyMs: number;
57
+ }
58
+ /**
59
+ * Drop `dropTimes` down to the ones inside the trailing window and cap the
60
+ * retained history. Always returns a new array.
61
+ */
62
+ declare function pruneDropTimes(dropTimes: unknown, nowMs?: number, windowMs?: number): number[];
63
+ /**
64
+ * Append one drop and return the pruned history. The same millisecond is
65
+ * collapsed (a close + keep-alive trip for the same socket is one event).
66
+ */
67
+ declare function recordDrop(dropTimes: unknown, nowMs?: number, windowMs?: number): number[];
68
+ /**
69
+ * Decayed drop weight: 1.0 for a drop that just happened, 0.5 after one
70
+ * half-life, ~0 after a few. Entries outside the window are ignored.
71
+ */
72
+ declare function flapScore(dropTimes: unknown, nowMs?: number, halfLifeMs?: number, windowMs?: number): number;
73
+ /** Effective-latency penalty in ms for the node's recent drops. */
74
+ declare function flapPenaltyMs(dropTimes: unknown, options?: FlapScoreOptions): number;
75
+ /**
76
+ * Latency + decayed flap penalty, the value node selection sorts on.
77
+ * Unknown latency stays `Infinity` so it never outranks a measured node.
78
+ */
79
+ declare function effectiveLatencyMs(latencyMs: number | null | undefined, dropTimes: unknown, options?: FlapScoreOptions): number;
80
+ /**
81
+ * Churn counters for logging/persistence: exact trailing-window counts plus
82
+ * the decayed score used for ranking.
83
+ */
84
+ declare function dropCounters(dropTimes: unknown, options?: FlapScoreOptions): DropCounters;
85
+ export { DEFAULT_FLAP_HALF_LIFE_MS, DEFAULT_FLAP_PENALTY_MAX_MS, DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_WINDOW_MS, DROP_RATE_WINDOW_MS, MAX_DROP_TIMES, dropCounters, effectiveLatencyMs, flapPenaltyMs, flapScore, pruneDropTimes, recordDrop, };
86
+ export type { DropCounters, FlapScoreOptions };
87
+ //# sourceMappingURL=node_flap_score.d.ts.map
@@ -0,0 +1,124 @@
1
+ 'use strict';
2
+ /**
3
+ * modules/node_flap_score.ts - Time-decayed connection-churn (flapping) scoring
4
+ *
5
+ * Node selection used to rank nodes by raw RPC latency only, so the
6
+ * lowest-ping node won even when it dropped its socket every few hours. Live
7
+ * logs showed exactly that: three Cloudflare-fronted endpoints answered in
8
+ * 50-120ms and accounted for every abnormal close, while three
9
+ * 300-360ms endpoints never dropped once — and were never selected.
10
+ *
11
+ * This module turns transport drop *timestamps* into a ranking input:
12
+ *
13
+ * effectiveLatency = latencyMs + flapPenaltyMs
14
+ * flapScore = Σ 0.5 ^ (age_i / halfLife)
15
+ * flapPenaltyMs = min(flapScore * DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_PENALTY_MAX_MS)
16
+ *
17
+ * A node with no recent drops ranks on its ping alone. One undecayed drop costs
18
+ * DEFAULT_FLAP_PENALTY_MS_PER_DROP of effective latency and then fades with the
19
+ * half-life, so a node that flaps once and then behaves is re-admitted
20
+ * automatically, a node that keeps flapping stays at the back of the list, and
21
+ * nothing has to be "reset" by a lucky health probe (the old
22
+ * LIVE_FAILURE_HEALTH_SUCCESS_STREAK wipe-on-success counter).
23
+ *
24
+ * Everything here is pure: callers own persistence and logging. Drop history
25
+ * is bounded by the window (24h default) and capped by MAX_DROP_TIMES, so it
26
+ * is safe to persist in the shared health cache (see node_health_cache.ts).
27
+ */
28
+ /** Trailing window of drops that may influence selection. */
29
+ const DEFAULT_FLAP_WINDOW_MS = 24 * 60 * 60 * 1000;
30
+ /** Time for a drop's penalty weight to halve. */
31
+ const DEFAULT_FLAP_HALF_LIFE_MS = 12 * 60 * 60 * 1000;
32
+ /** Effective-latency cost of one *undecayed* drop. */
33
+ const DEFAULT_FLAP_PENALTY_MS_PER_DROP = 5_000;
34
+ /** Ceiling for the decayed penalty, so a badly degraded node stays usable as a last resort. */
35
+ const DEFAULT_FLAP_PENALTY_MAX_MS = 10 * 60 * 1000;
36
+ /** Hard cap on retained drop timestamps per node (bounded persistence). */
37
+ const MAX_DROP_TIMES = 64;
38
+ /** Trailing window used for the reported drops/hour counters. */
39
+ const DROP_RATE_WINDOW_MS = 60 * 60 * 1000;
40
+ function positiveOr(value, fallback) {
41
+ const n = Number(value);
42
+ return Number.isFinite(n) && n > 0 ? n : fallback;
43
+ }
44
+ /** Drop timestamps that are valid finite numbers, ascending. */
45
+ function normalizeDropTimes(dropTimes) {
46
+ if (!Array.isArray(dropTimes))
47
+ return [];
48
+ return dropTimes
49
+ .filter((t) => typeof t === 'number' && Number.isFinite(t) && t > 0)
50
+ .sort((a, b) => a - b);
51
+ }
52
+ /**
53
+ * Drop `dropTimes` down to the ones inside the trailing window and cap the
54
+ * retained history. Always returns a new array.
55
+ */
56
+ function pruneDropTimes(dropTimes, nowMs = Date.now(), windowMs = DEFAULT_FLAP_WINDOW_MS) {
57
+ const window = positiveOr(windowMs, DEFAULT_FLAP_WINDOW_MS);
58
+ const cutoff = nowMs - window;
59
+ const kept = normalizeDropTimes(dropTimes).filter((t) => t > cutoff && t <= nowMs);
60
+ return kept.length > MAX_DROP_TIMES ? kept.slice(kept.length - MAX_DROP_TIMES) : kept;
61
+ }
62
+ /**
63
+ * Append one drop and return the pruned history. The same millisecond is
64
+ * collapsed (a close + keep-alive trip for the same socket is one event).
65
+ */
66
+ function recordDrop(dropTimes, nowMs = Date.now(), windowMs = DEFAULT_FLAP_WINDOW_MS) {
67
+ const pruned = pruneDropTimes(dropTimes, nowMs, windowMs);
68
+ if (pruned.length > 0 && pruned[pruned.length - 1] === nowMs)
69
+ return pruned;
70
+ return pruneDropTimes([...pruned, nowMs], nowMs, windowMs);
71
+ }
72
+ /**
73
+ * Decayed drop weight: 1.0 for a drop that just happened, 0.5 after one
74
+ * half-life, ~0 after a few. Entries outside the window are ignored.
75
+ */
76
+ function flapScore(dropTimes, nowMs = Date.now(), halfLifeMs = DEFAULT_FLAP_HALF_LIFE_MS, windowMs = DEFAULT_FLAP_WINDOW_MS) {
77
+ const halfLife = positiveOr(halfLifeMs, DEFAULT_FLAP_HALF_LIFE_MS);
78
+ let score = 0;
79
+ for (const t of pruneDropTimes(dropTimes, nowMs, windowMs)) {
80
+ score += Math.pow(0.5, (nowMs - t) / halfLife);
81
+ }
82
+ return score;
83
+ }
84
+ /** Effective-latency penalty in ms for the node's recent drops. */
85
+ function flapPenaltyMs(dropTimes, options = {}) {
86
+ const nowMs = Number.isFinite(options.nowMs) ? options.nowMs : Date.now();
87
+ const score = flapScore(dropTimes, nowMs, options.halfLifeMs, options.windowMs);
88
+ if (score <= 0)
89
+ return 0;
90
+ const perDrop = positiveOr(options.penaltyMsPerDrop, DEFAULT_FLAP_PENALTY_MS_PER_DROP);
91
+ const maxPenalty = positiveOr(options.maxPenaltyMs, DEFAULT_FLAP_PENALTY_MAX_MS);
92
+ return Math.min(score * perDrop, maxPenalty);
93
+ }
94
+ /**
95
+ * Latency + decayed flap penalty, the value node selection sorts on.
96
+ * Unknown latency stays `Infinity` so it never outranks a measured node.
97
+ */
98
+ function effectiveLatencyMs(latencyMs, dropTimes, options = {}) {
99
+ if (typeof latencyMs !== 'number' || !Number.isFinite(latencyMs))
100
+ return Infinity;
101
+ return latencyMs + flapPenaltyMs(dropTimes, options);
102
+ }
103
+ /**
104
+ * Churn counters for logging/persistence: exact trailing-window counts plus
105
+ * the decayed score used for ranking.
106
+ */
107
+ function dropCounters(dropTimes, options = {}) {
108
+ const nowMs = Number.isFinite(options.nowMs) ? options.nowMs : Date.now();
109
+ const windowMs = positiveOr(options.windowMs, DEFAULT_FLAP_WINDOW_MS);
110
+ const pruned = pruneDropTimes(dropTimes, nowMs, windowMs);
111
+ const dropsLastHour = pruned.filter((t) => nowMs - t <= DROP_RATE_WINDOW_MS).length;
112
+ const score = flapScore(pruned, nowMs, options.halfLifeMs, windowMs);
113
+ return {
114
+ dropsLastHour,
115
+ drops24h: pruned.length,
116
+ // Round to 2 decimals: a 24h average is meaningless beyond that and the
117
+ // value is persisted in JSON.
118
+ dropsPerHour: Math.round((pruned.length / (windowMs / (60 * 60 * 1000))) * 100) / 100,
119
+ flapScore: Math.round(score * 1000) / 1000,
120
+ penaltyMs: Math.round(flapPenaltyMs(pruned, { ...options, nowMs })),
121
+ };
122
+ }
123
+ export { DEFAULT_FLAP_HALF_LIFE_MS, DEFAULT_FLAP_PENALTY_MAX_MS, DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_WINDOW_MS, DROP_RATE_WINDOW_MS, MAX_DROP_TIMES, dropCounters, effectiveLatencyMs, flapPenaltyMs, flapScore, pruneDropTimes, recordDrop, };
124
+ //# sourceMappingURL=node_flap_score.js.map
@@ -3,18 +3,44 @@ interface HealthCacheOptions {
3
3
  stateDir?: string;
4
4
  now?: number;
5
5
  maxAgeMs?: number;
6
+ /**
7
+ * Flap-scoring tuning of the writer/reader. Defaults to
8
+ * modules/node_flap_score.ts values (which modules/constants.ts also reuses),
9
+ * so an operator who tuned `NODES.healthCheck.flap*` gets counters and
10
+ * ordering that match their bot's own ranking instead of the stock maths.
11
+ */
12
+ flap?: {
13
+ windowMs?: number;
14
+ halfLifeMs?: number;
15
+ penaltyMsPerDrop?: number;
16
+ maxPenaltyMs?: number;
17
+ };
6
18
  }
7
19
  interface NodeHealthStat {
8
20
  url?: string;
9
21
  status?: string;
10
22
  latencyMs?: number;
11
23
  lastCheckTime?: string | number | null;
24
+ /** Transport drop timestamps (see modules/node_flap_score.ts). */
25
+ dropTimes?: number[];
12
26
  }
13
27
  interface NodeHealthCacheEntry {
14
28
  url: string;
15
29
  status: string;
16
30
  latencyMs: number | null;
17
31
  lastCheckTime: string | number | null;
32
+ /**
33
+ * Connection churn, persisted so every consumer (bots, credential daemon)
34
+ * can see stability and not just ping. `dropTimes` is the source of truth
35
+ * for decay; the counters are derived for readers that only look at the
36
+ * JSON.
37
+ */
38
+ dropTimes: number[];
39
+ flapScore: number;
40
+ flapPenaltyMs: number;
41
+ dropsLastHour: number;
42
+ drops24h: number;
43
+ dropsPerHour: number;
18
44
  }
19
45
  interface NodeHealthCachePayload {
20
46
  version: number;
@@ -34,21 +60,32 @@ interface NodeSettings {
34
60
  }
35
61
  declare function resolveHealthCacheFile(config?: HealthCacheOptions): string;
36
62
  /**
37
- * Comparator for healthy/slow nodes: healthy first, then by ascending latency.
63
+ * Comparator for healthy/slow nodes: healthy first, then by *effective*
64
+ * latency (measured ping + decayed connection-churn penalty).
38
65
  * Shared by node_health_cache and node_manager.
39
66
  */
40
67
  declare function compareNodeHealth(a: {
41
68
  status: string;
42
69
  latencyMs?: number | null;
70
+ dropTimes?: number[];
71
+ flapPenaltyMs?: number;
43
72
  }, b: {
44
73
  status: string;
45
74
  latencyMs?: number | null;
46
- }): number;
75
+ dropTimes?: number[];
76
+ flapPenaltyMs?: number;
77
+ }, nowMs?: number, flap?: HealthCacheOptions['flap']): number;
78
+ /** Ranking latency: measured ping plus the churn penalty, Infinity when unknown. */
79
+ declare function nodeRankLatency(node: {
80
+ latencyMs?: number | null;
81
+ dropTimes?: number[];
82
+ flapPenaltyMs?: number;
83
+ }, nowMs: number, flap?: HealthCacheOptions['flap']): number;
47
84
  declare function writeHealthCache(stats: Iterable<NodeHealthStat> | ArrayLike<NodeHealthStat> | null | undefined, options?: HealthCacheOptions): NodeHealthCachePayload;
48
85
  declare function readHealthCache(options?: HealthCacheOptions): NodeHealthCachePayload | null;
49
86
  declare function removeNodesFromHealthCache(urls: string | string[], options?: HealthCacheOptions): NodeHealthCachePayload | null;
50
87
  declare function orderNodesFromHealthCache(configuredNodes: unknown, options?: HealthCacheOptions): string[];
51
88
  declare function resolveConfiguredNodes(settings: NodeSettings): string[];
52
89
  declare function orderNodesForSettings(settings: NodeSettings, options?: HealthCacheOptions): string[];
53
- export { compareNodeHealth, orderNodesFromHealthCache, orderNodesForSettings, readHealthCache, removeNodesFromHealthCache, resolveConfiguredNodes, resolveHealthCacheFile, writeHealthCache };
90
+ export { compareNodeHealth, nodeRankLatency, orderNodesFromHealthCache, orderNodesForSettings, readHealthCache, removeNodesFromHealthCache, resolveConfiguredNodes, resolveHealthCacheFile, writeHealthCache };
54
91
  //# sourceMappingURL=node_health_cache.d.ts.map