dexbot 1.6.6 → 1.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -1
- package/README.md +8 -5
- package/analysis/README.md +2 -3
- package/analysis/tradingview/README.md +78 -1
- package/analysis/trend_detection/DYNAMIC_WEIGHT_RESEARCH.md +1 -1
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +4 -0
- package/dist/analysis/tradingview/analyze_tradingview.js +55 -0
- package/dist/analysis/tradingview/grid_reset_config.d.ts +61 -0
- package/dist/analysis/tradingview/grid_reset_config.js +235 -0
- package/dist/analysis/tradingview/grid_reset_sim.d.ts +67 -0
- package/dist/analysis/tradingview/grid_reset_sim.js +307 -0
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +419 -3
- package/dist/dexbot.js +44 -15
- package/dist/market_adapter/core/market_adapter_service.js +14 -2
- package/dist/market_adapter/core/strategies/dynamic_weight_series.d.ts +8 -1
- package/dist/market_adapter/core/strategies/dynamic_weight_series.js +5 -1
- package/dist/market_adapter/inputs/window_cache.d.ts +97 -19
- package/dist/market_adapter/inputs/window_cache.js +302 -48
- package/dist/market_adapter/market_adapter.d.ts +8 -1
- package/dist/market_adapter/market_adapter.js +22 -4
- package/dist/modules/account_bots.js +20 -9
- package/dist/modules/account_orders.d.ts +44 -4
- package/dist/modules/account_orders.js +104 -7
- package/dist/modules/bitshares-native/chain_client.d.ts +10 -0
- package/dist/modules/bitshares-native/chain_client.js +197 -28
- package/dist/modules/bitshares-native/subscriptions.js +312 -10
- package/dist/modules/bitshares-native/transport.d.ts +1 -0
- package/dist/modules/bitshares-native/transport.js +97 -3
- package/dist/modules/bitshares_client.d.ts +20 -0
- package/dist/modules/bitshares_client.js +29 -0
- package/dist/modules/bot_settings.js +10 -1
- package/dist/modules/chain_keys.d.ts +37 -22
- package/dist/modules/chain_keys.js +145 -44
- package/dist/modules/cli_start_onboarding.d.ts +10 -0
- package/dist/modules/cli_start_onboarding.js +15 -0
- package/dist/modules/cli_start_output.d.ts +6 -0
- package/dist/modules/cli_start_output.js +8 -0
- package/dist/modules/config.d.ts +1 -0
- package/dist/modules/config.js +1 -0
- package/dist/modules/constants.d.ts +23 -2
- package/dist/modules/constants.js +196 -15
- package/dist/modules/dexbot_class.d.ts +6 -76
- package/dist/modules/dexbot_class.js +0 -32
- package/dist/modules/dexbot_cow_runtime.d.ts +3 -58
- package/dist/modules/dexbot_cow_runtime.js +53 -96
- package/dist/modules/dexbot_fill_runtime.d.ts +6 -1
- package/dist/modules/dexbot_fill_runtime.js +22 -3
- package/dist/modules/dexbot_maintenance_runtime.d.ts +38 -1
- package/dist/modules/dexbot_maintenance_runtime.js +103 -11
- package/dist/modules/dexbot_startup_runtime.js +73 -20
- package/dist/modules/dexbot_state_recovery.d.ts +6 -0
- package/dist/modules/dexbot_state_recovery.js +34 -1
- package/dist/modules/order/async_lock.d.ts +8 -0
- package/dist/modules/order/async_lock.js +15 -0
- package/dist/modules/order/genesis_policy.d.ts +162 -0
- package/dist/modules/order/genesis_policy.js +267 -0
- package/dist/modules/order/grid.js +109 -135
- package/dist/modules/order/grid_reconcile.d.ts +9 -0
- package/dist/modules/order/grid_reconcile.js +13 -4
- package/dist/modules/order/grid_reconcile_internal.js +13 -2
- package/dist/modules/order/logger.d.ts +12 -2
- package/dist/modules/order/logger.js +29 -7
- package/dist/modules/order/manager.d.ts +53 -0
- package/dist/modules/order/manager.js +86 -25
- package/dist/modules/order/sync_engine.d.ts +19 -0
- package/dist/modules/order/sync_engine.js +284 -344
- package/dist/modules/order/utils/math.d.ts +22 -1
- package/dist/modules/order/utils/math.js +64 -1
- package/dist/modules/order/utils/order.d.ts +43 -58
- package/dist/modules/order/utils/order.js +302 -194
- package/dist/modules/order/utils/system.d.ts +97 -0
- package/dist/modules/order/utils/system.js +282 -3
- package/dist/modules/paths.d.ts +8 -1
- package/dist/modules/paths.js +11 -3
- package/dist/modules/utils/text_width.d.ts +19 -0
- package/dist/modules/utils/text_width.js +97 -0
- package/dist/modules/version_notice.d.ts +165 -0
- package/dist/modules/version_notice.js +364 -0
- package/dist/pm2.js +104 -18
- package/dist/scripts/update.js +62 -13
- package/dist/unlock.d.ts +14 -2
- package/dist/unlock.js +101 -4
- package/docs/BITSHARES_ONBOARDING.md +8 -1
- package/docs/COW_INVARIANTS.md +11 -2
- package/docs/CREDENTIAL_SECURITY.md +26 -0
- package/docs/DEXBOT_COMPARISON.md +4 -4
- package/docs/EVOLUTION.md +20 -18
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_PRICE_INVARIANT.md +204 -31
- package/docs/GRID_RECALCULATION.md +12 -0
- package/docs/LOGGING.md +17 -2
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +6 -7
- package/docs/README.md +3 -1
- package/docs/WORKFLOW.md +16 -5
- package/market_adapter/README.md +1 -1
- package/modules/README.md +18 -1
- package/package.json +14 -6
- package/scripts/README.md +28 -2
- package/scripts/clear-all.sh +13 -4
- package/scripts/clear-logs.sh +13 -4
- package/scripts/clear-market-adapter.sh +6 -0
- package/scripts/clear-orders.sh +5 -0
- package/scripts/lib/dexbot-paths.sh +108 -0
- package/tests/README.md +11 -3
- package/analysis/account_resolver.ts +0 -161
- package/analysis/ama_fitting/analyze_ama_price_changes.ts +0 -155
- package/analysis/ama_fitting/analyze_lambda_vs_slow.ts +0 -404
- package/analysis/ama_fitting/calibrate_convergence_er.ts +0 -201
- package/analysis/ama_fitting/fetch_lp_candles.ts +0 -184
- package/analysis/ama_fitting/generate_unified_comparison_chart.ts +0 -232
- package/analysis/ama_fitting/optimizer_high_resolution.ts +0 -802
- package/analysis/analyze_dynamic_weight.ts +0 -243
- package/analysis/analyze_kalman.ts +0 -157
- package/analysis/analyze_regime.ts +0 -158
- package/analysis/analyze_regime_windows.ts +0 -470
- package/analysis/analyze_risk_profile.ts +0 -199
- package/analysis/analyze_trade_heatmap.ts +0 -362
- package/analysis/analyze_volatility.ts +0 -182
- package/analysis/bot_fitting/backtest_ama_sweep.ts +0 -934
- package/analysis/bot_fitting/backtest_bot_fitting.ts +0 -861
- package/analysis/bot_fitting/shared_utils.ts +0 -52
- package/analysis/bot_key_utils.ts +0 -426
- package/analysis/bot_usage/discover_bot_accounts.ts +0 -635
- package/analysis/bot_usage/kibana_bot_queries.ts +0 -211
- package/analysis/chain_pool.ts +0 -47
- package/analysis/chart_css.ts +0 -62
- package/analysis/chart_ui.ts +0 -105
- package/analysis/chart_utils.ts +0 -210
- package/analysis/fills_source.ts +0 -228
- package/analysis/grid_correction_check.ts +0 -786
- package/analysis/math_utils.ts +0 -53
- package/analysis/price_sources.ts +0 -136
- package/analysis/resolve_bot_accounts.ts +0 -199
- package/analysis/resolve_source.ts +0 -99
- package/analysis/results/ama_sweep_results_lp_pool_133_1h.json +0 -2455
- package/analysis/results/bot_fitting_results_lp_pool_133_1h.json +0 -218
- package/analysis/trade_profitability.ts +0 -1335
- package/analysis/tradingview/analyze_tradingview.ts +0 -280
- package/analysis/tradingview/h-bts_tradingview.html +0 -1570
- package/analysis/tradingview/t-bts_tradingview.html +0 -1570
- package/analysis/tradingview/tradingview_uplot_chart_generator.ts +0 -3268
- package/analysis/trend_detection/dynamic_weight_chart_generator.ts +0 -1481
- package/analysis/trend_detection/hurst_analyzer.ts +0 -7
- package/analysis/trend_detection/kalman_chart_generator.ts +0 -398
- package/analysis/trend_detection/kalman_trend_analyzer.ts +0 -9
- package/analysis/trend_detection/kalman_velocity_smoothing.ts +0 -7
- package/analysis/trend_detection/package.json +0 -23
- package/analysis/trend_detection/permutation_entropy_analyzer.ts +0 -7
- package/analysis/trend_detection/regime_chart_generator.ts +0 -338
- package/analysis/trend_detection/tests/test_kalman_trend.ts +0 -153
- package/analysis/trend_detection/tests/test_kalman_velocity_smoothing.ts +0 -64
- package/analysis/trend_detection/volatility_chart_generator.ts +0 -654
- package/analysis/tsconfig.json +0 -29
- package/claw/tests/package.json +0 -3
- package/claw/tests/test_claw_bridge.ts +0 -570
- package/claw/tests/test_claw_catalog_and_credentials.ts +0 -156
- package/claw/tests/test_claw_chain_layer.ts +0 -739
- package/claw/tests/test_claw_data_flow.ts +0 -525
- package/claw/tests/test_claw_domain_logic.ts +0 -471
- package/claw/tests/test_claw_manifest_and_matrix.ts +0 -222
- package/claw/tests/test_claw_mcp_transport.ts +0 -243
- package/claw/tests/test_claw_regressions.ts +0 -711
- package/claw/tests/test_claw_skill_generation.ts +0 -103
- package/claw/tests/test_claw_state_store.ts +0 -44
- package/claw/tests/test_dexbot_profiles.ts +0 -654
- package/claw/tests/test_memu_bridge.ts +0 -189
- package/claw/tests/test_nullclaw_tmp_integration.ts +0 -57
- package/claw/tests/test_position_health.ts +0 -343
- package/claw/tests/test_position_manager.ts +0 -429
- package/claw/tests/test_position_manager_watch_health.ts +0 -100
- package/claw/tests/test_short_mpa_strategy.ts +0 -232
- package/dist/analysis/account_resolver.d.ts.map +0 -1
- package/dist/analysis/account_resolver.js.map +0 -1
- package/dist/analysis/ama_fitting/analyze_ama_price_changes.d.ts.map +0 -1
- package/dist/analysis/ama_fitting/analyze_ama_price_changes.js.map +0 -1
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.d.ts.map +0 -1
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js.map +0 -1
- package/dist/analysis/ama_fitting/calibrate_convergence_er.d.ts.map +0 -1
- package/dist/analysis/ama_fitting/calibrate_convergence_er.js.map +0 -1
- package/dist/analysis/ama_fitting/fetch_lp_candles.d.ts.map +0 -1
- package/dist/analysis/ama_fitting/fetch_lp_candles.js.map +0 -1
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts.map +0 -1
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js.map +0 -1
- package/dist/analysis/ama_fitting/optimizer_high_resolution.d.ts.map +0 -1
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js.map +0 -1
- package/dist/analysis/analyze_dynamic_weight.d.ts.map +0 -1
- package/dist/analysis/analyze_dynamic_weight.js.map +0 -1
- package/dist/analysis/analyze_kalman.d.ts.map +0 -1
- package/dist/analysis/analyze_kalman.js.map +0 -1
- package/dist/analysis/analyze_regime.d.ts.map +0 -1
- package/dist/analysis/analyze_regime.js.map +0 -1
- package/dist/analysis/analyze_regime_windows.d.ts.map +0 -1
- package/dist/analysis/analyze_regime_windows.js.map +0 -1
- package/dist/analysis/analyze_risk_profile.d.ts.map +0 -1
- package/dist/analysis/analyze_risk_profile.js.map +0 -1
- package/dist/analysis/analyze_trade_heatmap.d.ts.map +0 -1
- package/dist/analysis/analyze_trade_heatmap.js.map +0 -1
- package/dist/analysis/analyze_volatility.d.ts.map +0 -1
- package/dist/analysis/analyze_volatility.js.map +0 -1
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts.map +0 -1
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js.map +0 -1
- package/dist/analysis/bot_fitting/backtest_bot_fitting.d.ts.map +0 -1
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js.map +0 -1
- package/dist/analysis/bot_fitting/shared_utils.d.ts.map +0 -1
- package/dist/analysis/bot_fitting/shared_utils.js.map +0 -1
- package/dist/analysis/bot_key_utils.d.ts.map +0 -1
- package/dist/analysis/bot_key_utils.js.map +0 -1
- package/dist/analysis/bot_usage/discover_bot_accounts.d.ts.map +0 -1
- package/dist/analysis/bot_usage/discover_bot_accounts.js.map +0 -1
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts.map +0 -1
- package/dist/analysis/bot_usage/kibana_bot_queries.js.map +0 -1
- package/dist/analysis/chain_pool.d.ts.map +0 -1
- package/dist/analysis/chain_pool.js.map +0 -1
- package/dist/analysis/chart_css.d.ts.map +0 -1
- package/dist/analysis/chart_css.js.map +0 -1
- package/dist/analysis/chart_ui.d.ts.map +0 -1
- package/dist/analysis/chart_ui.js.map +0 -1
- package/dist/analysis/chart_utils.d.ts.map +0 -1
- package/dist/analysis/chart_utils.js.map +0 -1
- package/dist/analysis/fills_source.d.ts.map +0 -1
- package/dist/analysis/fills_source.js.map +0 -1
- package/dist/analysis/grid_correction_check.d.ts.map +0 -1
- package/dist/analysis/grid_correction_check.js.map +0 -1
- package/dist/analysis/math_utils.d.ts.map +0 -1
- package/dist/analysis/math_utils.js.map +0 -1
- package/dist/analysis/price_sources.d.ts.map +0 -1
- package/dist/analysis/price_sources.js.map +0 -1
- package/dist/analysis/resolve_bot_accounts.d.ts.map +0 -1
- package/dist/analysis/resolve_bot_accounts.js.map +0 -1
- package/dist/analysis/resolve_source.d.ts.map +0 -1
- package/dist/analysis/resolve_source.js.map +0 -1
- package/dist/analysis/trade_profitability.d.ts.map +0 -1
- package/dist/analysis/trade_profitability.js.map +0 -1
- package/dist/analysis/tradingview/analyze_tradingview.d.ts.map +0 -1
- package/dist/analysis/tradingview/analyze_tradingview.js.map +0 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts.map +0 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js.map +0 -1
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.d.ts.map +0 -1
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js.map +0 -1
- package/dist/analysis/trend_detection/hurst_analyzer.d.ts.map +0 -1
- package/dist/analysis/trend_detection/hurst_analyzer.js.map +0 -1
- package/dist/analysis/trend_detection/kalman_chart_generator.d.ts.map +0 -1
- package/dist/analysis/trend_detection/kalman_chart_generator.js.map +0 -1
- package/dist/analysis/trend_detection/kalman_trend_analyzer.d.ts.map +0 -1
- package/dist/analysis/trend_detection/kalman_trend_analyzer.js.map +0 -1
- package/dist/analysis/trend_detection/kalman_velocity_smoothing.d.ts.map +0 -1
- package/dist/analysis/trend_detection/kalman_velocity_smoothing.js.map +0 -1
- package/dist/analysis/trend_detection/permutation_entropy_analyzer.d.ts.map +0 -1
- package/dist/analysis/trend_detection/permutation_entropy_analyzer.js.map +0 -1
- package/dist/analysis/trend_detection/regime_chart_generator.d.ts.map +0 -1
- package/dist/analysis/trend_detection/regime_chart_generator.js.map +0 -1
- package/dist/analysis/trend_detection/tests/test_kalman_trend.d.ts +0 -2
- package/dist/analysis/trend_detection/tests/test_kalman_trend.d.ts.map +0 -1
- package/dist/analysis/trend_detection/tests/test_kalman_trend.js +0 -128
- package/dist/analysis/trend_detection/tests/test_kalman_trend.js.map +0 -1
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.d.ts +0 -2
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.d.ts.map +0 -1
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.js +0 -37
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.js.map +0 -1
- package/dist/analysis/trend_detection/volatility_chart_generator.d.ts.map +0 -1
- package/dist/analysis/trend_detection/volatility_chart_generator.js.map +0 -1
- package/dist/bot.d.ts.map +0 -1
- package/dist/bot.js.map +0 -1
- package/dist/credential-daemon.d.ts.map +0 -1
- package/dist/credential-daemon.js.map +0 -1
- package/dist/dexbot.d.ts.map +0 -1
- package/dist/dexbot.js.map +0 -1
- package/dist/market_adapter/ama_signal_runner.d.ts.map +0 -1
- package/dist/market_adapter/ama_signal_runner.js.map +0 -1
- package/dist/market_adapter/candle_utils.d.ts.map +0 -1
- package/dist/market_adapter/candle_utils.js.map +0 -1
- package/dist/market_adapter/core/asymmetric_bounds.d.ts.map +0 -1
- package/dist/market_adapter/core/asymmetric_bounds.js.map +0 -1
- package/dist/market_adapter/core/config_normalizers.d.ts.map +0 -1
- package/dist/market_adapter/core/config_normalizers.js.map +0 -1
- package/dist/market_adapter/core/kibana_candles.d.ts.map +0 -1
- package/dist/market_adapter/core/kibana_candles.js.map +0 -1
- package/dist/market_adapter/core/kibana_client.d.ts.map +0 -1
- package/dist/market_adapter/core/kibana_client.js.map +0 -1
- package/dist/market_adapter/core/kibana_market_candles.d.ts.map +0 -1
- package/dist/market_adapter/core/kibana_market_candles.js.map +0 -1
- package/dist/market_adapter/core/market_adapter_service.d.ts.map +0 -1
- package/dist/market_adapter/core/market_adapter_service.js.map +0 -1
- package/dist/market_adapter/core/signals/hurst_analyzer.d.ts.map +0 -1
- package/dist/market_adapter/core/signals/hurst_analyzer.js.map +0 -1
- package/dist/market_adapter/core/signals/kalman_trend_analyzer.d.ts.map +0 -1
- package/dist/market_adapter/core/signals/kalman_trend_analyzer.js.map +0 -1
- package/dist/market_adapter/core/signals/kalman_velocity_smoothing.d.ts.map +0 -1
- package/dist/market_adapter/core/signals/kalman_velocity_smoothing.js.map +0 -1
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts.map +0 -1
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js.map +0 -1
- package/dist/market_adapter/core/strategies/ama.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/ama.js.map +0 -1
- package/dist/market_adapter/core/strategies/ama_slope_model.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/ama_slope_model.js.map +0 -1
- package/dist/market_adapter/core/strategies/atr/calculator.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/atr/calculator.js.map +0 -1
- package/dist/market_adapter/core/strategies/collateral_manager.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/collateral_manager.js.map +0 -1
- package/dist/market_adapter/core/strategies/dynamic_weight_series.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/dynamic_weight_series.js.map +0 -1
- package/dist/market_adapter/core/strategies/regime_gate.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/regime_gate.js.map +0 -1
- package/dist/market_adapter/core/strategies/regime_interp.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/regime_interp.js.map +0 -1
- package/dist/market_adapter/core/strategies/volatility_shift.d.ts.map +0 -1
- package/dist/market_adapter/core/strategies/volatility_shift.js.map +0 -1
- package/dist/market_adapter/inputs/fetch_book_data.d.ts.map +0 -1
- package/dist/market_adapter/inputs/fetch_book_data.js.map +0 -1
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.d.ts.map +0 -1
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js.map +0 -1
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts.map +0 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js.map +0 -1
- package/dist/market_adapter/inputs/kibana_feed_source.d.ts.map +0 -1
- package/dist/market_adapter/inputs/kibana_feed_source.js.map +0 -1
- package/dist/market_adapter/inputs/kibana_source.d.ts.map +0 -1
- package/dist/market_adapter/inputs/kibana_source.js.map +0 -1
- package/dist/market_adapter/inputs/window_cache.d.ts.map +0 -1
- package/dist/market_adapter/inputs/window_cache.js.map +0 -1
- package/dist/market_adapter/interval_utils.d.ts.map +0 -1
- package/dist/market_adapter/interval_utils.js.map +0 -1
- package/dist/market_adapter/log_format.d.ts.map +0 -1
- package/dist/market_adapter/log_format.js.map +0 -1
- package/dist/market_adapter/lp_chart_core.d.ts.map +0 -1
- package/dist/market_adapter/lp_chart_core.js.map +0 -1
- package/dist/market_adapter/lp_chart_runner.d.ts.map +0 -1
- package/dist/market_adapter/lp_chart_runner.js.map +0 -1
- package/dist/market_adapter/lp_chart_strategy_loader.d.ts.map +0 -1
- package/dist/market_adapter/lp_chart_strategy_loader.js.map +0 -1
- package/dist/market_adapter/market_adapter.d.ts.map +0 -1
- package/dist/market_adapter/market_adapter.js.map +0 -1
- package/dist/market_adapter/test_helpers.d.ts.map +0 -1
- package/dist/market_adapter/test_helpers.js.map +0 -1
- package/dist/market_adapter/utils/adapter_client.d.ts.map +0 -1
- package/dist/market_adapter/utils/adapter_client.js.map +0 -1
- package/dist/market_adapter/utils/atomic_write.d.ts.map +0 -1
- package/dist/market_adapter/utils/atomic_write.js.map +0 -1
- package/dist/market_adapter/utils/chain.d.ts.map +0 -1
- package/dist/market_adapter/utils/chain.js.map +0 -1
- package/dist/market_adapter/utils/data_discovery.d.ts.map +0 -1
- package/dist/market_adapter/utils/data_discovery.js.map +0 -1
- package/dist/market_adapter/utils/dynamic_grid_snapshot.d.ts.map +0 -1
- package/dist/market_adapter/utils/dynamic_grid_snapshot.js.map +0 -1
- package/dist/market_adapter/utils/file_lock.d.ts.map +0 -1
- package/dist/market_adapter/utils/file_lock.js.map +0 -1
- package/dist/market_adapter/utils/native_history.d.ts.map +0 -1
- package/dist/market_adapter/utils/native_history.js.map +0 -1
- package/dist/modules/account_bots.d.ts.map +0 -1
- package/dist/modules/account_bots.js.map +0 -1
- package/dist/modules/account_orders.d.ts.map +0 -1
- package/dist/modules/account_orders.js.map +0 -1
- package/dist/modules/authority_resolver.d.ts.map +0 -1
- package/dist/modules/authority_resolver.js.map +0 -1
- package/dist/modules/bitshares-native/chain_client.d.ts.map +0 -1
- package/dist/modules/bitshares-native/chain_client.js.map +0 -1
- package/dist/modules/bitshares-native/crypto/ecc.browser.d.ts.map +0 -1
- package/dist/modules/bitshares-native/crypto/ecc.browser.js.map +0 -1
- package/dist/modules/bitshares-native/crypto/ecc.d.ts.map +0 -1
- package/dist/modules/bitshares-native/crypto/ecc.js.map +0 -1
- package/dist/modules/bitshares-native/crypto/ecc_selector.d.ts.map +0 -1
- package/dist/modules/bitshares-native/crypto/ecc_selector.js.map +0 -1
- package/dist/modules/bitshares-native/index.d.ts.map +0 -1
- package/dist/modules/bitshares-native/index.js.map +0 -1
- package/dist/modules/bitshares-native/lru_cache.d.ts.map +0 -1
- package/dist/modules/bitshares-native/lru_cache.js.map +0 -1
- package/dist/modules/bitshares-native/resolvers.d.ts.map +0 -1
- package/dist/modules/bitshares-native/resolvers.js.map +0 -1
- package/dist/modules/bitshares-native/serial/chain_constants.d.ts.map +0 -1
- package/dist/modules/bitshares-native/serial/chain_constants.js.map +0 -1
- package/dist/modules/bitshares-native/serial/index.d.ts.map +0 -1
- package/dist/modules/bitshares-native/serial/index.js.map +0 -1
- package/dist/modules/bitshares-native/serial/operations.d.ts.map +0 -1
- package/dist/modules/bitshares-native/serial/operations.js.map +0 -1
- package/dist/modules/bitshares-native/serial/serializer.d.ts.map +0 -1
- package/dist/modules/bitshares-native/serial/serializer.js.map +0 -1
- package/dist/modules/bitshares-native/serial/types.d.ts.map +0 -1
- package/dist/modules/bitshares-native/serial/types.js.map +0 -1
- package/dist/modules/bitshares-native/signing_client.d.ts.map +0 -1
- package/dist/modules/bitshares-native/signing_client.js.map +0 -1
- package/dist/modules/bitshares-native/subscriptions.d.ts.map +0 -1
- package/dist/modules/bitshares-native/subscriptions.js.map +0 -1
- package/dist/modules/bitshares-native/transport.d.ts.map +0 -1
- package/dist/modules/bitshares-native/transport.js.map +0 -1
- package/dist/modules/bitshares-native/tx/builder.d.ts.map +0 -1
- package/dist/modules/bitshares-native/tx/builder.js.map +0 -1
- package/dist/modules/bitshares-native/tx/tx_cache.d.ts.map +0 -1
- package/dist/modules/bitshares-native/tx/tx_cache.js.map +0 -1
- package/dist/modules/bitshares_client.d.ts.map +0 -1
- package/dist/modules/bitshares_client.js.map +0 -1
- package/dist/modules/bot_defaults.d.ts.map +0 -1
- package/dist/modules/bot_defaults.js.map +0 -1
- package/dist/modules/bot_settings.d.ts.map +0 -1
- package/dist/modules/bot_settings.js.map +0 -1
- package/dist/modules/bots_file_lock.d.ts.map +0 -1
- package/dist/modules/bots_file_lock.js.map +0 -1
- package/dist/modules/broadcast_failure.d.ts.map +0 -1
- package/dist/modules/broadcast_failure.js.map +0 -1
- package/dist/modules/chain_keys.d.ts.map +0 -1
- package/dist/modules/chain_keys.js.map +0 -1
- package/dist/modules/chain_orders.d.ts.map +0 -1
- package/dist/modules/chain_orders.js.map +0 -1
- package/dist/modules/cli_colors.d.ts.map +0 -1
- package/dist/modules/cli_colors.js.map +0 -1
- package/dist/modules/config.d.ts.map +0 -1
- package/dist/modules/config.js.map +0 -1
- package/dist/modules/constants.d.ts.map +0 -1
- package/dist/modules/constants.js.map +0 -1
- package/dist/modules/cr_planner.d.ts.map +0 -1
- package/dist/modules/cr_planner.js.map +0 -1
- package/dist/modules/credential_policy.d.ts.map +0 -1
- package/dist/modules/credential_policy.js.map +0 -1
- package/dist/modules/credential_runtime.d.ts.map +0 -1
- package/dist/modules/credential_runtime.js.map +0 -1
- package/dist/modules/credential_session_cache.d.ts.map +0 -1
- package/dist/modules/credential_session_cache.js.map +0 -1
- package/dist/modules/credit_pricing.d.ts.map +0 -1
- package/dist/modules/credit_pricing.js.map +0 -1
- package/dist/modules/credit_runtime.d.ts.map +0 -1
- package/dist/modules/credit_runtime.js.map +0 -1
- package/dist/modules/crypto/browser_provider.d.ts.map +0 -1
- package/dist/modules/crypto/browser_provider.js.map +0 -1
- package/dist/modules/crypto/index.d.ts.map +0 -1
- package/dist/modules/crypto/index.js.map +0 -1
- package/dist/modules/crypto/node_provider.d.ts.map +0 -1
- package/dist/modules/crypto/node_provider.js.map +0 -1
- package/dist/modules/crypto/provider.d.ts.map +0 -1
- package/dist/modules/crypto/provider.js.map +0 -1
- package/dist/modules/crypto/pure_ripemd160.d.ts.map +0 -1
- package/dist/modules/crypto/pure_ripemd160.js.map +0 -1
- package/dist/modules/crypto/pure_scrypt.d.ts.map +0 -1
- package/dist/modules/crypto/pure_scrypt.js.map +0 -1
- package/dist/modules/crypto/pure_secp256k1.d.ts.map +0 -1
- package/dist/modules/crypto/pure_secp256k1.js.map +0 -1
- package/dist/modules/crypto/sync.d.ts.map +0 -1
- package/dist/modules/crypto/sync.js.map +0 -1
- package/dist/modules/daemon_node_health.d.ts.map +0 -1
- package/dist/modules/daemon_node_health.js.map +0 -1
- package/dist/modules/dexbot_class.d.ts.map +0 -1
- package/dist/modules/dexbot_class.js.map +0 -1
- package/dist/modules/dexbot_cow_runtime.d.ts.map +0 -1
- package/dist/modules/dexbot_cow_runtime.js.map +0 -1
- package/dist/modules/dexbot_credential_client.d.ts.map +0 -1
- package/dist/modules/dexbot_credential_client.js.map +0 -1
- package/dist/modules/dexbot_fill_runtime.d.ts.map +0 -1
- package/dist/modules/dexbot_fill_runtime.js.map +0 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +0 -1
- package/dist/modules/dexbot_maintenance_runtime.js.map +0 -1
- package/dist/modules/dexbot_startup_runtime.d.ts.map +0 -1
- package/dist/modules/dexbot_startup_runtime.js.map +0 -1
- package/dist/modules/dexbot_state_recovery.d.ts.map +0 -1
- package/dist/modules/dexbot_state_recovery.js.map +0 -1
- package/dist/modules/env.d.ts.map +0 -1
- package/dist/modules/env.js.map +0 -1
- package/dist/modules/fund_registry.d.ts.map +0 -1
- package/dist/modules/fund_registry.js.map +0 -1
- package/dist/modules/general_settings.d.ts.map +0 -1
- package/dist/modules/general_settings.js.map +0 -1
- package/dist/modules/graceful_shutdown.d.ts.map +0 -1
- package/dist/modules/graceful_shutdown.js.map +0 -1
- package/dist/modules/grid_price_source.d.ts.map +0 -1
- package/dist/modules/grid_price_source.js.map +0 -1
- package/dist/modules/key_store.d.ts.map +0 -1
- package/dist/modules/key_store.js.map +0 -1
- package/dist/modules/launcher/adapter_requirement.d.ts.map +0 -1
- package/dist/modules/launcher/adapter_requirement.js.map +0 -1
- package/dist/modules/launcher/bot_supervisor.d.ts.map +0 -1
- package/dist/modules/launcher/bot_supervisor.js.map +0 -1
- package/dist/modules/launcher/child_env.d.ts.map +0 -1
- package/dist/modules/launcher/child_env.js.map +0 -1
- package/dist/modules/launcher/credential_bootstrap.d.ts.map +0 -1
- package/dist/modules/launcher/credential_bootstrap.js.map +0 -1
- package/dist/modules/launcher/credential_daemon.d.ts.map +0 -1
- package/dist/modules/launcher/credential_daemon.js.map +0 -1
- package/dist/modules/launcher/credential_secret.d.ts.map +0 -1
- package/dist/modules/launcher/credential_secret.js.map +0 -1
- package/dist/modules/launcher/foreign_cred_daemon.d.ts.map +0 -1
- package/dist/modules/launcher/foreign_cred_daemon.js.map +0 -1
- package/dist/modules/launcher/headless_password.d.ts.map +0 -1
- package/dist/modules/launcher/headless_password.js.map +0 -1
- package/dist/modules/launcher/launch_modes.d.ts.map +0 -1
- package/dist/modules/launcher/launch_modes.js.map +0 -1
- package/dist/modules/launcher/market_adapter_runtime.d.ts.map +0 -1
- package/dist/modules/launcher/market_adapter_runtime.js.map +0 -1
- package/dist/modules/launcher/market_adapter_watchdog.d.ts.map +0 -1
- package/dist/modules/launcher/market_adapter_watchdog.js.map +0 -1
- package/dist/modules/launcher/monolithic_runtime.d.ts.map +0 -1
- package/dist/modules/launcher/monolithic_runtime.js.map +0 -1
- package/dist/modules/launcher/runtime_entry.d.ts.map +0 -1
- package/dist/modules/launcher/runtime_entry.js.map +0 -1
- package/dist/modules/launcher/status_reporting.d.ts.map +0 -1
- package/dist/modules/launcher/status_reporting.js.map +0 -1
- package/dist/modules/launcher/supervisor_control.d.ts.map +0 -1
- package/dist/modules/launcher/supervisor_control.js.map +0 -1
- package/dist/modules/market_adapter_whitelist.d.ts.map +0 -1
- package/dist/modules/market_adapter_whitelist.js.map +0 -1
- package/dist/modules/node_connect_policy.d.ts.map +0 -1
- package/dist/modules/node_connect_policy.js.map +0 -1
- package/dist/modules/node_failure_ledger.d.ts.map +0 -1
- package/dist/modules/node_failure_ledger.js.map +0 -1
- package/dist/modules/node_health_cache.d.ts.map +0 -1
- package/dist/modules/node_health_cache.js.map +0 -1
- package/dist/modules/node_manager.d.ts.map +0 -1
- package/dist/modules/node_manager.js.map +0 -1
- package/dist/modules/order/accounting.d.ts.map +0 -1
- package/dist/modules/order/accounting.js.map +0 -1
- package/dist/modules/order/async_lock.d.ts.map +0 -1
- package/dist/modules/order/async_lock.js.map +0 -1
- package/dist/modules/order/export.d.ts.map +0 -1
- package/dist/modules/order/export.js.map +0 -1
- package/dist/modules/order/format.d.ts.map +0 -1
- package/dist/modules/order/format.js.map +0 -1
- package/dist/modules/order/grid.d.ts.map +0 -1
- package/dist/modules/order/grid.js.map +0 -1
- package/dist/modules/order/grid_reconcile.d.ts.map +0 -1
- package/dist/modules/order/grid_reconcile.js.map +0 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts.map +0 -1
- package/dist/modules/order/grid_reconcile_internal.js.map +0 -1
- package/dist/modules/order/index.d.ts.map +0 -1
- package/dist/modules/order/index.js.map +0 -1
- package/dist/modules/order/logger.d.ts.map +0 -1
- package/dist/modules/order/logger.js.map +0 -1
- package/dist/modules/order/logger_state.d.ts.map +0 -1
- package/dist/modules/order/logger_state.js.map +0 -1
- package/dist/modules/order/manager.d.ts.map +0 -1
- package/dist/modules/order/manager.js.map +0 -1
- package/dist/modules/order/processed_fill_store.d.ts.map +0 -1
- package/dist/modules/order/processed_fill_store.js.map +0 -1
- package/dist/modules/order/strategy.d.ts.map +0 -1
- package/dist/modules/order/strategy.js.map +0 -1
- package/dist/modules/order/sync_engine.d.ts.map +0 -1
- package/dist/modules/order/sync_engine.js.map +0 -1
- package/dist/modules/order/utils/math.d.ts.map +0 -1
- package/dist/modules/order/utils/math.js.map +0 -1
- package/dist/modules/order/utils/order.d.ts.map +0 -1
- package/dist/modules/order/utils/order.js.map +0 -1
- package/dist/modules/order/utils/slot.d.ts.map +0 -1
- package/dist/modules/order/utils/slot.js.map +0 -1
- package/dist/modules/order/utils/system.d.ts.map +0 -1
- package/dist/modules/order/utils/system.js.map +0 -1
- package/dist/modules/order/utils/timeout.d.ts.map +0 -1
- package/dist/modules/order/utils/timeout.js.map +0 -1
- package/dist/modules/order/utils/validate.d.ts.map +0 -1
- package/dist/modules/order/utils/validate.js.map +0 -1
- package/dist/modules/order/utils/withPoolRef.d.ts.map +0 -1
- package/dist/modules/order/utils/withPoolRef.js.map +0 -1
- package/dist/modules/order/working_grid.d.ts.map +0 -1
- package/dist/modules/order/working_grid.js.map +0 -1
- package/dist/modules/path_api.d.ts.map +0 -1
- package/dist/modules/path_api.js.map +0 -1
- package/dist/modules/paths.d.ts.map +0 -1
- package/dist/modules/paths.js.map +0 -1
- package/dist/modules/process_discovery.d.ts.map +0 -1
- package/dist/modules/process_discovery.js.map +0 -1
- package/dist/modules/runtime.d.ts.map +0 -1
- package/dist/modules/runtime.js.map +0 -1
- package/dist/modules/runtime_settings.d.ts.map +0 -1
- package/dist/modules/runtime_settings.js.map +0 -1
- package/dist/modules/settings_merge.d.ts.map +0 -1
- package/dist/modules/settings_merge.js.map +0 -1
- package/dist/modules/socket_json_client.d.ts.map +0 -1
- package/dist/modules/socket_json_client.js.map +0 -1
- package/dist/modules/storage/browser_adapter.d.ts.map +0 -1
- package/dist/modules/storage/browser_adapter.js.map +0 -1
- package/dist/modules/storage/index.d.ts.map +0 -1
- package/dist/modules/storage/index.js.map +0 -1
- package/dist/modules/storage/node_adapter.d.ts.map +0 -1
- package/dist/modules/storage/node_adapter.js.map +0 -1
- package/dist/modules/storage/types.d.ts.map +0 -1
- package/dist/modules/storage/types.js.map +0 -1
- package/dist/modules/types.d.ts.map +0 -1
- package/dist/modules/types.js.map +0 -1
- package/dist/modules/utils/base58check.d.ts.map +0 -1
- package/dist/modules/utils/base58check.js.map +0 -1
- package/dist/modules/utils/build_dir.d.ts.map +0 -1
- package/dist/modules/utils/build_dir.js.map +0 -1
- package/dist/modules/utils/chain_logs.d.ts.map +0 -1
- package/dist/modules/utils/chain_logs.js.map +0 -1
- package/dist/modules/utils/errors.d.ts.map +0 -1
- package/dist/modules/utils/errors.js.map +0 -1
- package/dist/modules/utils/sanitize_key.d.ts.map +0 -1
- package/dist/modules/utils/sanitize_key.js.map +0 -1
- package/dist/modules/validate_profiles.d.ts.map +0 -1
- package/dist/modules/validate_profiles.js.map +0 -1
- package/dist/pm2.d.ts.map +0 -1
- package/dist/pm2.js.map +0 -1
- package/dist/scripts/analyze-credit.d.ts.map +0 -1
- package/dist/scripts/analyze-credit.js.map +0 -1
- package/dist/scripts/analyze-git.d.ts.map +0 -1
- package/dist/scripts/analyze-git.js.map +0 -1
- package/dist/scripts/analyze-orders.d.ts.map +0 -1
- package/dist/scripts/analyze-orders.js.map +0 -1
- package/dist/scripts/chart_command.d.ts.map +0 -1
- package/dist/scripts/chart_command.js.map +0 -1
- package/dist/scripts/diagnose-kibana-candles.d.ts.map +0 -1
- package/dist/scripts/diagnose-kibana-candles.js.map +0 -1
- package/dist/scripts/diagnose-pool-history.d.ts.map +0 -1
- package/dist/scripts/diagnose-pool-history.js.map +0 -1
- package/dist/scripts/divergence-calc.d.ts.map +0 -1
- package/dist/scripts/divergence-calc.js.map +0 -1
- package/dist/scripts/dw.d.ts.map +0 -1
- package/dist/scripts/dw.js.map +0 -1
- package/dist/scripts/generate_lp_chart.d.ts.map +0 -1
- package/dist/scripts/generate_lp_chart.js.map +0 -1
- package/dist/scripts/generate_mainnet_corpus_report.d.ts.map +0 -1
- package/dist/scripts/generate_mainnet_corpus_report.js.map +0 -1
- package/dist/scripts/native_release_gates.d.ts.map +0 -1
- package/dist/scripts/native_release_gates.js.map +0 -1
- package/dist/scripts/print_grid.d.ts.map +0 -1
- package/dist/scripts/print_grid.js.map +0 -1
- package/dist/scripts/run-tests.d.ts.map +0 -1
- package/dist/scripts/run-tests.js.map +0 -1
- package/dist/scripts/runner.d.ts.map +0 -1
- package/dist/scripts/runner.js.map +0 -1
- package/dist/scripts/sync-version.d.ts.map +0 -1
- package/dist/scripts/sync-version.js.map +0 -1
- package/dist/scripts/test-credit-renewal.d.ts.map +0 -1
- package/dist/scripts/test-credit-renewal.js.map +0 -1
- package/dist/scripts/tv.d.ts.map +0 -1
- package/dist/scripts/tv.js.map +0 -1
- package/dist/scripts/update.d.ts.map +0 -1
- package/dist/scripts/update.js.map +0 -1
- package/dist/scripts/update_dist_freshness.d.ts.map +0 -1
- package/dist/scripts/update_dist_freshness.js.map +0 -1
- package/dist/scripts/validate_bots.d.ts.map +0 -1
- package/dist/scripts/validate_bots.js.map +0 -1
- package/dist/scripts/verify-browser-bundle.d.ts.map +0 -1
- package/dist/scripts/verify-browser-bundle.js.map +0 -1
- package/dist/unlock.d.ts.map +0 -1
- package/dist/unlock.js.map +0 -1
|
@@ -32,6 +32,35 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
32
32
|
// Fill polling timer + re-entrancy guard for discovering fills via history scan.
|
|
33
33
|
let fillPollTimer = null;
|
|
34
34
|
let fillPollInProgress = false;
|
|
35
|
+
// Fill-channel health. A history channel can be dead (stale api id, wedged
|
|
36
|
+
// session) while the socket still reads "connected", so the transport's
|
|
37
|
+
// close/keep-alive recovery never fires. These settings drive a per-account
|
|
38
|
+
// consecutive-failure watchdog that forces a reconnect when the channel
|
|
39
|
+
// stays dead, plus log throttling so a dead channel cannot flood the log.
|
|
40
|
+
const channelDegradedThreshold = Number.isFinite(SUBSCRIPTIONS.CHANNEL_DEGRADED_FAILURE_THRESHOLD)
|
|
41
|
+
? Math.max(1, SUBSCRIPTIONS.CHANNEL_DEGRADED_FAILURE_THRESHOLD)
|
|
42
|
+
: 3;
|
|
43
|
+
const channelErrorLogIntervalMs = Number.isFinite(overrides?.channelErrorLogIntervalMs)
|
|
44
|
+
? Math.max(0, overrides.channelErrorLogIntervalMs)
|
|
45
|
+
: (Number.isFinite(SUBSCRIPTIONS.CHANNEL_ERROR_LOG_INTERVAL_MS)
|
|
46
|
+
? Math.max(0, SUBSCRIPTIONS.CHANNEL_ERROR_LOG_INTERVAL_MS)
|
|
47
|
+
: 60000);
|
|
48
|
+
// Test seams so offline tests never sleep on the retry-ladder delays.
|
|
49
|
+
const channelRetryLadderMs = (Array.isArray(overrides?.channelRetryLadderMs)
|
|
50
|
+
? overrides.channelRetryLadderMs
|
|
51
|
+
: (Array.isArray(SUBSCRIPTIONS.CHANNEL_RETRY_LADDER_MS) ? SUBSCRIPTIONS.CHANNEL_RETRY_LADDER_MS : [5000, 10000, 15000]))
|
|
52
|
+
.map((d) => Math.max(0, Number(d) || 0))
|
|
53
|
+
.filter((d) => d > 0);
|
|
54
|
+
const channelRecoveryAlertAfter = Number.isFinite(SUBSCRIPTIONS.CHANNEL_RECOVERY_ALERT_AFTER)
|
|
55
|
+
? Math.max(1, SUBSCRIPTIONS.CHANNEL_RECOVERY_ALERT_AFTER)
|
|
56
|
+
: 3;
|
|
57
|
+
const channelRetryMaxRefills = Number.isFinite(overrides?.channelRetryMaxRefills)
|
|
58
|
+
? Math.max(0, overrides.channelRetryMaxRefills)
|
|
59
|
+
: (Number.isFinite(SUBSCRIPTIONS.CHANNEL_RETRY_MAX_REFILLS) ? Math.max(0, SUBSCRIPTIONS.CHANNEL_RETRY_MAX_REFILLS) : 3);
|
|
60
|
+
// NOTE: the forced-reconnect cooldown is NOT debounced here. It is enforced
|
|
61
|
+
// once per client in chain_client.forceReconnect so this watchdog and the
|
|
62
|
+
// stale api_id escalation share a single window (a wedged session trips both
|
|
63
|
+
// counters in the same tick and must not stack two reconnects).
|
|
35
64
|
function parseObjectIdInstance(id) {
|
|
36
65
|
if (typeof id !== 'string')
|
|
37
66
|
return Number.NaN;
|
|
@@ -85,10 +114,260 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
85
114
|
return 'unknown node';
|
|
86
115
|
}
|
|
87
116
|
}
|
|
88
|
-
function warnSubscription(sub, message, err = null) {
|
|
117
|
+
function warnSubscription(sub, message, err = null, throttleKey = 'channel') {
|
|
89
118
|
const account = sub?.accountName || sub?.accountId || 'unknown';
|
|
119
|
+
const now = Date.now();
|
|
120
|
+
// Throttle per-account AND per-category. A dead channel otherwise logs
|
|
121
|
+
// one warn per poll forever; a permanently failing callback must not be
|
|
122
|
+
// masked by (or mask) channel-error lines. Suppressed repeats are
|
|
123
|
+
// counted per category and reported on the next emitted line.
|
|
124
|
+
let state = null;
|
|
125
|
+
if (sub) {
|
|
126
|
+
if (!sub._warnThrottle || typeof sub._warnThrottle !== 'object')
|
|
127
|
+
sub._warnThrottle = {};
|
|
128
|
+
state = sub._warnThrottle[throttleKey] || { lastAt: 0, suppressed: 0 };
|
|
129
|
+
}
|
|
130
|
+
if (channelErrorLogIntervalMs > 0 && sub && now - state.lastAt < channelErrorLogIntervalMs) {
|
|
131
|
+
state.suppressed += 1;
|
|
132
|
+
sub._warnThrottle[throttleKey] = state;
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
const suppressed = state ? (Number(state.suppressed) || 0) : 0;
|
|
136
|
+
if (sub)
|
|
137
|
+
sub._warnThrottle[throttleKey] = { lastAt: now, suppressed: 0 };
|
|
90
138
|
const detail = err?.message ? `: ${getErrorMessage(err)}` : '';
|
|
91
|
-
|
|
139
|
+
const suppressedDetail = suppressed > 0 ? ` (+${suppressed} suppressed)` : '';
|
|
140
|
+
subscriptionsLogger.warn(`${message} for ${account}${detail}${suppressedDetail} (node=${activeNodeUrl()})`);
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Mark a subscription's history channel healthy again. Resets the
|
|
144
|
+
* consecutive-failure run and the recovery-cycle counter, and logs a single
|
|
145
|
+
* recovery line when it was previously degraded.
|
|
146
|
+
*/
|
|
147
|
+
function recordChannelSuccess(sub) {
|
|
148
|
+
if (!sub)
|
|
149
|
+
return;
|
|
150
|
+
const account = sub.accountName || sub.accountId || 'unknown';
|
|
151
|
+
if (sub._channelDegraded || sub._channelRecoveryCycles > 0) {
|
|
152
|
+
const cycles = Number(sub._channelRecoveryCycles) || 0;
|
|
153
|
+
const after = cycles > 0 ? ` after ${cycles} forced reconnect(s)` : '';
|
|
154
|
+
subscriptionsLogger.info(`Fill channel recovered for ${account}${after} (node=${activeNodeUrl()})`);
|
|
155
|
+
}
|
|
156
|
+
clearChannelRetry(sub);
|
|
157
|
+
sub._channelFailures = 0;
|
|
158
|
+
sub._channelDegraded = false;
|
|
159
|
+
sub._channelRecoveryCycles = 0;
|
|
160
|
+
sub._recoveryAlerted = false;
|
|
161
|
+
sub._channelRetryStep = 0;
|
|
162
|
+
sub._channelRetryRefills = 0;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Schedule the next rung of the escalating re-scan ladder that follows a
|
|
166
|
+
* channel failure, so a recovery attempt is verified in seconds instead of
|
|
167
|
+
* after the next 60s fill-poll tick.
|
|
168
|
+
*
|
|
169
|
+
* The ladder is bounded: once exhausted the regular poll cadence takes over
|
|
170
|
+
* again, so a permanently dead channel cannot become a tight scan loop. The
|
|
171
|
+
* step index resets whenever a reconnect is actually issued (a new recovery
|
|
172
|
+
* attempt deserves a fresh verification ladder) and on any successful scan.
|
|
173
|
+
*/
|
|
174
|
+
function scheduleChannelRetry(sub, context) {
|
|
175
|
+
if (!sub || channelRetryLadderMs.length === 0)
|
|
176
|
+
return;
|
|
177
|
+
if (sub._channelRetryTimer)
|
|
178
|
+
return;
|
|
179
|
+
const step = Number(sub._channelRetryStep) || 0;
|
|
180
|
+
if (step >= channelRetryLadderMs.length)
|
|
181
|
+
return;
|
|
182
|
+
sub._channelRetryStep = step + 1;
|
|
183
|
+
// Label the rung distinctly from the scan that scheduled it, and include
|
|
184
|
+
// the rung number, so a log reader can tell a fast retry from a 60s poll
|
|
185
|
+
// tick and see how far the ladder has climbed.
|
|
186
|
+
const retryContext = `retry${sub._channelRetryStep}-after-${context || 'scan'}`;
|
|
187
|
+
const timer = setTimeout(() => {
|
|
188
|
+
sub._channelRetryTimer = null;
|
|
189
|
+
// The entry may have been unsubscribed (or already recovered) while
|
|
190
|
+
// the retry was pending.
|
|
191
|
+
if (!sub.active || !subscriptions.has(sub.accountName))
|
|
192
|
+
return;
|
|
193
|
+
if (!sub.accountId)
|
|
194
|
+
return;
|
|
195
|
+
// Never overlap another scan for the same sub. A regular poll, a
|
|
196
|
+
// notice/eager-gap scan, or a resubscribe may already be in flight;
|
|
197
|
+
// its result will schedule the next rung (success clears the ladder,
|
|
198
|
+
// failure re-enters recordChannelFailure), so dropping this rung is
|
|
199
|
+
// safe and mirrors the guards used by every other processObjects
|
|
200
|
+
// caller.
|
|
201
|
+
if (sub.reconnecting || sub._processingHistory || pendingScans.has(sub))
|
|
202
|
+
return;
|
|
203
|
+
sub._processingHistory = true;
|
|
204
|
+
processObjects(sub, [sub.accountId], { context: retryContext })
|
|
205
|
+
.catch((err) => {
|
|
206
|
+
// processObjects handles its own errors; this only guards
|
|
207
|
+
// against a throw from the retry plumbing itself.
|
|
208
|
+
recordChannelFailure(sub, err, retryContext);
|
|
209
|
+
})
|
|
210
|
+
.finally(() => {
|
|
211
|
+
sub._processingHistory = false;
|
|
212
|
+
});
|
|
213
|
+
}, channelRetryLadderMs[step]);
|
|
214
|
+
sub._channelRetryTimer = timer;
|
|
215
|
+
if (typeof timer?.unref === 'function')
|
|
216
|
+
timer.unref();
|
|
217
|
+
}
|
|
218
|
+
function clearChannelRetry(sub) {
|
|
219
|
+
if (sub?._channelRetryTimer) {
|
|
220
|
+
clearTimeout(sub._channelRetryTimer);
|
|
221
|
+
sub._channelRetryTimer = null;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Record a history-channel failure and keep escalating while the channel
|
|
226
|
+
* stays dead. Callback/processing errors (flagged subscriptionErrorReported)
|
|
227
|
+
* are not channel failures and must not trip the watchdog — a deterministic
|
|
228
|
+
* downstream bug would otherwise reconnect-storm.
|
|
229
|
+
*
|
|
230
|
+
* Reconnect requests are NOT latched on the degraded flag: if a forced
|
|
231
|
+
* reconnect fails to clear the channel (every node equally stale, or the
|
|
232
|
+
* same session bug on the new node), the account must keep asking. The rate
|
|
233
|
+
* is bounded centrally by the per-client cooldown in
|
|
234
|
+
* chain_client.forceReconnect, so re-arming here cannot storm. Only the
|
|
235
|
+
* DEGRADED log line and the operator alert are transition-gated.
|
|
236
|
+
*/
|
|
237
|
+
function recordChannelFailure(sub, err, context) {
|
|
238
|
+
if (!sub)
|
|
239
|
+
return;
|
|
240
|
+
const label = `processObjects${context ? ` (${context})` : ''}`;
|
|
241
|
+
if (err?.subscriptionErrorReported) {
|
|
242
|
+
warnSubscription(sub, `${label}: callback error`, err, 'callback');
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
const failures = (Number(sub._channelFailures) || 0) + 1;
|
|
246
|
+
sub._channelFailures = failures;
|
|
247
|
+
warnSubscription(sub, `${label}: error`, err);
|
|
248
|
+
const account = sub.accountName || sub.accountId || 'unknown';
|
|
249
|
+
if (failures >= channelDegradedThreshold) {
|
|
250
|
+
if (!sub._channelDegraded) {
|
|
251
|
+
sub._channelDegraded = true;
|
|
252
|
+
subscriptionsLogger.warn(`Fill channel DEGRADED for ${account}: ${failures} consecutive history-scan failures ` +
|
|
253
|
+
`(last: ${getErrorMessage(err)}) — forcing reconnect (node=${activeNodeUrl()})`);
|
|
254
|
+
}
|
|
255
|
+
const outcome = requestChannelReconnect(`fill channel degraded for ${account}: ${getErrorMessage(err)}`);
|
|
256
|
+
// Count a recovery cycle for a genuinely new teardown AND for one
|
|
257
|
+
// coalesced onto another escalation source's recent reconnect. The
|
|
258
|
+
// stale api_id path runs inside the RPC catch, before this failure
|
|
259
|
+
// reaches us, so without counting `coalesced` it would reliably
|
|
260
|
+
// consume the shared cooldown and starve the operator alert / node
|
|
261
|
+
// strike in exactly the stale-id wedge this watchdog targets.
|
|
262
|
+
if (outcome === 'issued' || outcome === 'coalesced') {
|
|
263
|
+
sub._channelRecoveryCycles = (Number(sub._channelRecoveryCycles) || 0) + 1;
|
|
264
|
+
// Only a genuinely new teardown gets a fresh verification ladder;
|
|
265
|
+
// a coalesced reconnect is already being verified by its issuer's
|
|
266
|
+
// own post-reconnect catch-up scan. Bounded by
|
|
267
|
+
// channelRetryMaxRefills so a dead channel cannot chain the ladder.
|
|
268
|
+
if (outcome === 'issued') {
|
|
269
|
+
const refills = Number(sub._channelRetryRefills) || 0;
|
|
270
|
+
if (refills < channelRetryMaxRefills) {
|
|
271
|
+
sub._channelRetryRefills = refills + 1;
|
|
272
|
+
sub._channelRetryStep = 0;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
// Recovery is not working. This is where the node earns a strike:
|
|
276
|
+
// the transport's own forced-reconnect report is deliberately
|
|
277
|
+
// kept out of the persistent ledger (see reportNodeFailureToManager
|
|
278
|
+
// in bitshares_client.ts), so a session-level wedge that clears
|
|
279
|
+
// on the first or second cycle costs the node nothing, while a
|
|
280
|
+
// node that survives repeated failed recovery is recorded here.
|
|
281
|
+
if (sub._channelRecoveryCycles >= channelRecoveryAlertAfter) {
|
|
282
|
+
reportChannelNodeFailure(account, err);
|
|
283
|
+
}
|
|
284
|
+
maybeAlertUnrecoverableChannel(sub, account, err);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
// Schedule the fast re-scan LAST, so the escalation above can refill the
|
|
288
|
+
// ladder first. Scheduling before it would consume the last rung on the
|
|
289
|
+
// very failure that escalates, leaving the fresh attempt with no retry
|
|
290
|
+
// scheduled and idling until the next 60s poll.
|
|
291
|
+
scheduleChannelRetry(sub, context);
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Record a REAL node failure once fill-channel recovery has demonstrably
|
|
295
|
+
* failed, so genuinely bad nodes still reach the blacklist threshold.
|
|
296
|
+
*
|
|
297
|
+
* Deliberately NOT done for every forced reconnect: the transport reports
|
|
298
|
+
* those as 'forced-reconnect' and bitshares_client.ts keeps that source out
|
|
299
|
+
* of the persistent strike ledger. A forced reconnect is an attempt; a node
|
|
300
|
+
* that is still serving a dead channel after several of them is evidence.
|
|
301
|
+
*
|
|
302
|
+
* Attribution: the strike lands on the node active at THIS failing scan, not
|
|
303
|
+
* necessarily the node that caused the original wedge — each forced
|
|
304
|
+
* reconnect rotates away from the failed node, so a per-cycle wedge spreads
|
|
305
|
+
* strikes across the nodes that keep serving it. A single bad node is
|
|
306
|
+
* rotated off and recovers before striking; a wedge shared by all nodes
|
|
307
|
+
* legitimately accumulates strikes against each of them.
|
|
308
|
+
*/
|
|
309
|
+
function reportChannelNodeFailure(account, err) {
|
|
310
|
+
const node = activeNodeUrl();
|
|
311
|
+
// 'unknown node' is the activeNodeUrl() fallback, not a reportable URL.
|
|
312
|
+
if (!node || node === 'unknown node')
|
|
313
|
+
return;
|
|
314
|
+
try {
|
|
315
|
+
if (typeof chainClient.reportNodeFailure === 'function') {
|
|
316
|
+
chainClient.reportNodeFailure(node, `fill channel unrecoverable for ${account}: ${getErrorMessage(err)}`, 'fill-channel-unrecoverable');
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
catch (_) {
|
|
320
|
+
// Best-effort: never let strike bookkeeping break the watchdog.
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Operator-facing escalation: after N forced reconnects without the channel
|
|
325
|
+
* recovering, recovery is not working and a human/restart is needed. Purely
|
|
326
|
+
* informational reconnect looping is invisible in the logs, which is how the
|
|
327
|
+
* original incident went unnoticed for hours.
|
|
328
|
+
*/
|
|
329
|
+
function maybeAlertUnrecoverableChannel(sub, account, err) {
|
|
330
|
+
if (sub._recoveryAlerted)
|
|
331
|
+
return;
|
|
332
|
+
const cycles = Number(sub._channelRecoveryCycles) || 0;
|
|
333
|
+
if (cycles < channelRecoveryAlertAfter)
|
|
334
|
+
return;
|
|
335
|
+
sub._recoveryAlerted = true;
|
|
336
|
+
subscriptionsLogger.error(`Fill channel for ${account} did NOT recover after ${cycles} forced reconnects ` +
|
|
337
|
+
`(last: ${getErrorMessage(err)}) — fills may be missed. Automatic recovery is giving up; ` +
|
|
338
|
+
`check node health, then restart the bot if the channel stays dead (node=${activeNodeUrl()})`);
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Force a reconnect to recover a dead-but-open fill channel. The debounce
|
|
342
|
+
* lives in chain_client.forceReconnect (one per client, shared with the
|
|
343
|
+
* stale api_id escalation), so this is a straight delegation.
|
|
344
|
+
*
|
|
345
|
+
* The reconnect re-establishes the login session AND fires the registered
|
|
346
|
+
* post-reconnect safety-net sync (the fill backlog is then discovered even
|
|
347
|
+
* if the subscription feed stays quiet).
|
|
348
|
+
*
|
|
349
|
+
* @returns the reconnect outcome: 'issued' (a new teardown started),
|
|
350
|
+
* 'coalesced' (a forced reconnect was already issued within the
|
|
351
|
+
* shared cooldown), or 'unavailable' (no live socket / no escalation
|
|
352
|
+
* path).
|
|
353
|
+
*/
|
|
354
|
+
function requestChannelReconnect(reason) {
|
|
355
|
+
try {
|
|
356
|
+
if (typeof chainClient.forceReconnect !== 'function')
|
|
357
|
+
return 'unavailable';
|
|
358
|
+
const outcome = chainClient.forceReconnect(reason);
|
|
359
|
+
// New chain_client contract. Legacy/mock booleans are normalized so
|
|
360
|
+
// the accounting is stable: true = issued, anything else = unavailable.
|
|
361
|
+
if (outcome === 'issued' || outcome === 'coalesced' || outcome === 'unavailable') {
|
|
362
|
+
return outcome;
|
|
363
|
+
}
|
|
364
|
+
return outcome === true ? 'issued' : 'unavailable';
|
|
365
|
+
}
|
|
366
|
+
catch (_) {
|
|
367
|
+
// Best-effort recovery; the next failure re-requests (the watchdog
|
|
368
|
+
// no longer latches, so a broken recovery is retried, not dropped).
|
|
369
|
+
return 'unavailable';
|
|
370
|
+
}
|
|
92
371
|
}
|
|
93
372
|
function getAccountHistoryFetcher() {
|
|
94
373
|
return chainClient.history?.getAccountHistory
|
|
@@ -599,6 +878,9 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
599
878
|
// Defensive: guarantee oldest-first ordering so the cursor advance below lands
|
|
600
879
|
// on the newest op (history[last]), never a gap entry pushed onto the end.
|
|
601
880
|
sortEntriesOldestFirst(history);
|
|
881
|
+
// The history RPC succeeded — the channel is alive regardless of what
|
|
882
|
+
// the entries contain or whether downstream callbacks succeed.
|
|
883
|
+
recordChannelSuccess(sub);
|
|
602
884
|
if (history.length === 0) {
|
|
603
885
|
sub.lastNoticeAt = Date.now();
|
|
604
886
|
subscriptionsLogger.debug(`processObjects: no history entries for ${sub.accountName} (cursor=${sub.lastDeliveredHistoryId})`);
|
|
@@ -633,7 +915,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
633
915
|
await Promise.resolve(callback(fills));
|
|
634
916
|
}
|
|
635
917
|
catch (err) {
|
|
636
|
-
|
|
918
|
+
warnSubscription(sub, 'processObjects: callback error', err, 'callback');
|
|
637
919
|
failed.push(err);
|
|
638
920
|
}
|
|
639
921
|
}
|
|
@@ -673,7 +955,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
673
955
|
}
|
|
674
956
|
catch (err) {
|
|
675
957
|
sub.lastNoticeAt = Date.now();
|
|
676
|
-
|
|
958
|
+
recordChannelFailure(sub, err, options?.context);
|
|
677
959
|
if (sub.onError && !err?.subscriptionErrorReported) {
|
|
678
960
|
try {
|
|
679
961
|
sub.onError(err);
|
|
@@ -696,7 +978,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
696
978
|
entry.reconnectRetryTimer = setTimeout(() => {
|
|
697
979
|
entry.reconnectRetryTimer = null;
|
|
698
980
|
resubscribeEntry(entry, 'retry').catch((retryErr) => {
|
|
699
|
-
|
|
981
|
+
warnSubscription(entry, 'Failed to resubscribe', retryErr);
|
|
700
982
|
scheduleReconnectRetry(entry, retryErr);
|
|
701
983
|
});
|
|
702
984
|
}, reconnectRetryDelayMs);
|
|
@@ -725,7 +1007,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
725
1007
|
await chainClient.db.get_full_accounts([subEntry.accountName], true);
|
|
726
1008
|
}
|
|
727
1009
|
catch (err) {
|
|
728
|
-
|
|
1010
|
+
warnSubscription(subEntry, 'Failed to re-subscribe account after set_subscribe_callback', err);
|
|
729
1011
|
failures.push({ entry: subEntry, err });
|
|
730
1012
|
}
|
|
731
1013
|
}
|
|
@@ -746,7 +1028,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
746
1028
|
}
|
|
747
1029
|
}
|
|
748
1030
|
catch (err) {
|
|
749
|
-
|
|
1031
|
+
warnSubscription(entry, 'Failed to refresh account data', err);
|
|
750
1032
|
}
|
|
751
1033
|
const refreshFailures = await refreshSubscriptions();
|
|
752
1034
|
const entryRefreshFailure = refreshFailures.find((failure) => failure.entry === entry);
|
|
@@ -787,6 +1069,11 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
787
1069
|
continue;
|
|
788
1070
|
if (entry._processingHistory)
|
|
789
1071
|
continue;
|
|
1072
|
+
// A probe is already scheduled for this account: it re-scans
|
|
1073
|
+
// in seconds, so a regular tick would only duplicate the
|
|
1074
|
+
// request and burn a failure count against the threshold.
|
|
1075
|
+
if (entry._channelRetryTimer)
|
|
1076
|
+
continue;
|
|
790
1077
|
// Poll is a lightweight liveness check (fetches only >cursor, ~1 page
|
|
791
1078
|
// in steady state). Gap recovery (2000 per-account ops ≈40 pages) is
|
|
792
1079
|
// already armed by handleNotice cursor advances and by
|
|
@@ -799,7 +1086,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
799
1086
|
// already handles the quiet-after-gap case in ~250ms, so this poll
|
|
800
1087
|
// is only the safety-net fallback.
|
|
801
1088
|
try {
|
|
802
|
-
await processObjects(entry, [entry.accountId]);
|
|
1089
|
+
await processObjects(entry, [entry.accountId], { context: 'fill-poll' });
|
|
803
1090
|
}
|
|
804
1091
|
catch (err) {
|
|
805
1092
|
subscriptionsLogger.warn(`Fill poll failed for ${entry.accountName}: ${getErrorMessage(err)}`);
|
|
@@ -842,6 +1129,20 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
842
1129
|
onError: null,
|
|
843
1130
|
reconnectRetryTimer: null,
|
|
844
1131
|
reconnecting: false,
|
|
1132
|
+
// Fill-channel health (see recordChannelFailure/recordChannelSuccess).
|
|
1133
|
+
_channelFailures: 0,
|
|
1134
|
+
_channelDegraded: false,
|
|
1135
|
+
// Forced reconnects issued for this account since the last
|
|
1136
|
+
// successful scan; drives the unrecoverable-channel alert.
|
|
1137
|
+
_channelRecoveryCycles: 0,
|
|
1138
|
+
_recoveryAlerted: false,
|
|
1139
|
+
// Pending fast re-scan after a failure: it retries in seconds, so
|
|
1140
|
+
// a regular tick would only duplicate the request and burn a
|
|
1141
|
+
// failure count against the threshold.
|
|
1142
|
+
_channelRetryTimer: null,
|
|
1143
|
+
_channelRetryStep: 0,
|
|
1144
|
+
_channelRetryRefills: 0,
|
|
1145
|
+
_warnThrottle: {},
|
|
845
1146
|
};
|
|
846
1147
|
subscriptions.set(accountName, entry);
|
|
847
1148
|
// Add callback BEFORE first await so a rollback unsubscribe()
|
|
@@ -948,6 +1249,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
948
1249
|
if (entry.callbacks.size === 0) {
|
|
949
1250
|
entry.active = false;
|
|
950
1251
|
clearReconnectRetry(entry);
|
|
1252
|
+
clearChannelRetry(entry);
|
|
951
1253
|
const pending = pendingScans.get(entry);
|
|
952
1254
|
if (pending) {
|
|
953
1255
|
if (pending.timer)
|
|
@@ -983,7 +1285,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
983
1285
|
entry.statisticsId = accounts[0][1].account.statistics || null;
|
|
984
1286
|
}
|
|
985
1287
|
}).catch((err) => {
|
|
986
|
-
|
|
1288
|
+
warnSubscription(entry, 'Failed to refresh account data', err);
|
|
987
1289
|
}));
|
|
988
1290
|
}
|
|
989
1291
|
await Promise.all(refreshTasks);
|
|
@@ -1012,7 +1314,7 @@ function createSubscriptionManager(chainClient, overrides = {}) {
|
|
|
1012
1314
|
scanTasks.push(processObjects(entry, [entry.accountId], { throwOnError: true })
|
|
1013
1315
|
.then(() => clearReconnectRetry(entry))
|
|
1014
1316
|
.catch((err) => {
|
|
1015
|
-
|
|
1317
|
+
warnSubscription(entry, 'Failed to resubscribe', err);
|
|
1016
1318
|
scheduleReconnectRetry(entry, err);
|
|
1017
1319
|
}));
|
|
1018
1320
|
}
|
|
@@ -31,6 +31,7 @@ interface TransportConfig {
|
|
|
31
31
|
declare function createTransport(config?: TransportConfig): {
|
|
32
32
|
connect: (servers?: string[], autoReconnect?: boolean) => Promise<void>;
|
|
33
33
|
disconnect: () => void;
|
|
34
|
+
forceReconnect: (reason?: string) => void;
|
|
34
35
|
call: (method: string, params: any[], timeoutMs?: number) => Promise<any>;
|
|
35
36
|
addMessageHandler: (handler: (params: any) => void) => (() => void) & {
|
|
36
37
|
isActive: () => boolean;
|
|
@@ -75,6 +75,10 @@ function createTransport(config = {}) {
|
|
|
75
75
|
let autoreconnect = false;
|
|
76
76
|
let intentionalClose = false;
|
|
77
77
|
let reconnectAttempts = 0;
|
|
78
|
+
// Single-flight connect. A forced reconnect can land while a
|
|
79
|
+
// scheduleReconnect() timer is already sweeping the node list; without this
|
|
80
|
+
// guard both sweeps run concurrently, open two sockets and orphan one.
|
|
81
|
+
let connectInFlight = null;
|
|
78
82
|
const maxReconnectAttempts = 20;
|
|
79
83
|
// Close-event debounce for the currently active socket only. Stale sockets
|
|
80
84
|
// are ignored by identity so a fresh socket close can never be suppressed by
|
|
@@ -180,7 +184,7 @@ function createTransport(config = {}) {
|
|
|
180
184
|
setStatus('closed');
|
|
181
185
|
reconnectTimer = setTimeout(() => {
|
|
182
186
|
reconnectTimer = null;
|
|
183
|
-
|
|
187
|
+
startConnect().catch(() => scheduleReconnect());
|
|
184
188
|
}, SLOW_RECONNECT_INTERVAL_MS);
|
|
185
189
|
return;
|
|
186
190
|
}
|
|
@@ -188,7 +192,7 @@ function createTransport(config = {}) {
|
|
|
188
192
|
reconnectAttempts++;
|
|
189
193
|
reconnectTimer = setTimeout(() => {
|
|
190
194
|
reconnectTimer = null;
|
|
191
|
-
|
|
195
|
+
startConnect().catch(() => scheduleReconnect());
|
|
192
196
|
}, delay);
|
|
193
197
|
}
|
|
194
198
|
function startKeepAlive() {
|
|
@@ -474,6 +478,30 @@ function createTransport(config = {}) {
|
|
|
474
478
|
setStatus('closed');
|
|
475
479
|
throw new AllNodesFailed(lastConnectErrors);
|
|
476
480
|
}
|
|
481
|
+
/**
|
|
482
|
+
* Run a connect sweep, coalescing concurrent callers onto one attempt.
|
|
483
|
+
* Returns the in-flight sweep when one is already running so a forced
|
|
484
|
+
* reconnect never races a scheduled reconnect into parallel sockets.
|
|
485
|
+
*
|
|
486
|
+
* `supersede` is for the explicit connect() path, which has just called
|
|
487
|
+
* disconnect() and therefore genuinely wants a fresh sweep against the
|
|
488
|
+
* CURRENT node list — inheriting a sweep that started under the old list
|
|
489
|
+
* could land it on a node the caller just replaced.
|
|
490
|
+
*/
|
|
491
|
+
function startConnect(supersede = false) {
|
|
492
|
+
if (!supersede && connectInFlight)
|
|
493
|
+
return connectInFlight;
|
|
494
|
+
const attempt = tryConnect();
|
|
495
|
+
connectInFlight = attempt;
|
|
496
|
+
// Swallow on the tracking copy only: callers still see the rejection via
|
|
497
|
+
// the returned promise, and this keeps the stored handle from ever
|
|
498
|
+
// becoming an unhandled rejection.
|
|
499
|
+
attempt.catch(() => { }).finally(() => {
|
|
500
|
+
if (connectInFlight === attempt)
|
|
501
|
+
connectInFlight = null;
|
|
502
|
+
});
|
|
503
|
+
return attempt;
|
|
504
|
+
}
|
|
477
505
|
async function connect(servers, autoReconnect = false) {
|
|
478
506
|
if (Array.isArray(servers)) {
|
|
479
507
|
nodeList = servers.filter(s => s && typeof s === 'string');
|
|
@@ -509,7 +537,7 @@ function createTransport(config = {}) {
|
|
|
509
537
|
disconnect();
|
|
510
538
|
autoreconnect = autoReconnect;
|
|
511
539
|
intentionalClose = false;
|
|
512
|
-
await
|
|
540
|
+
await startConnect(true);
|
|
513
541
|
}
|
|
514
542
|
function disconnect() {
|
|
515
543
|
intentionalClose = true;
|
|
@@ -540,6 +568,71 @@ function createTransport(config = {}) {
|
|
|
540
568
|
nodeUrl = null;
|
|
541
569
|
setStatus('closed');
|
|
542
570
|
}
|
|
571
|
+
/**
|
|
572
|
+
* Force a reconnect on the active connection, preferring a different node.
|
|
573
|
+
*
|
|
574
|
+
* The normal close/keep-alive paths only observe a *socket* failure. A
|
|
575
|
+
* wedged login session (e.g. cached api ids rejected with
|
|
576
|
+
* `_local_apis.size() > api_id` after a server-side session swap) leaves
|
|
577
|
+
* the socket open while every namespaced RPC fails, so nothing ever tears
|
|
578
|
+
* it down. This reports the active node as failed (so shouldSkipNode /
|
|
579
|
+
* the failure ledger deprioritizes it) and closes the socket with
|
|
580
|
+
* autoreconnect armed, letting the reconnect land on a healthy node.
|
|
581
|
+
*
|
|
582
|
+
* No-op when there is no active socket.
|
|
583
|
+
* @param {string} reason - Human-readable trigger for logs/metrics
|
|
584
|
+
*/
|
|
585
|
+
function forceReconnect(reason = 'forced') {
|
|
586
|
+
if (!ws || !nodeUrl)
|
|
587
|
+
return;
|
|
588
|
+
const failedNode = nodeUrl;
|
|
589
|
+
const oldSocket = ws;
|
|
590
|
+
// A forced reconnect is a fresh connection attempt, not a continuation
|
|
591
|
+
// of the backoff from a prior failure, so it must count as a reconnect
|
|
592
|
+
// (attempt > 0) for _onConnected to fire onReconnect (subscription
|
|
593
|
+
// re-establishment + post-reconnect safety-net sync).
|
|
594
|
+
//
|
|
595
|
+
// Math.max, not `= 1`: never lower an attempt count that is already
|
|
596
|
+
// elevated, only guarantee it is at least 1.
|
|
597
|
+
//
|
|
598
|
+
// NOTE: today this is an invariant guard rather than a behavior change.
|
|
599
|
+
// forceReconnect() is a no-op unless a socket is live, and the first
|
|
600
|
+
// failed reconnect attempt nulls `ws`, so it can only ever run right
|
|
601
|
+
// after _onConnected reset the counter to 0 — where Math.max(x, 1) and
|
|
602
|
+
// `= 1` are identical. It matters only if cleanup()/attemptConnect ever
|
|
603
|
+
// stop nulling `ws`, which would make a forced reconnect land mid-backoff
|
|
604
|
+
// and clamp the exponential growth back to the minimum delay.
|
|
605
|
+
reconnectAttempts = Math.max(reconnectAttempts, 1);
|
|
606
|
+
autoreconnect = true;
|
|
607
|
+
intentionalClose = false;
|
|
608
|
+
// Report once here; the teardown below detaches the old close handler so
|
|
609
|
+
// the node is never double-struck for this event.
|
|
610
|
+
notifyNodeFailure(failedNode, `forced reconnect: ${reason}`, 'forced-reconnect');
|
|
611
|
+
transportLogger.warn(`Forcing reconnect on ${failedNode} (${reason})`);
|
|
612
|
+
// Tear down synchronously instead of waiting for the close handshake:
|
|
613
|
+
// an unresponsive peer may never send its close frame, which is exactly
|
|
614
|
+
// the wedged-session case this recovery exists for. Detach onclose first
|
|
615
|
+
// so our own close() cannot schedule a second, duplicate reconnect.
|
|
616
|
+
oldSocket.onclose = null;
|
|
617
|
+
// A pending reconnectTimer is ours to clear (it would open a second
|
|
618
|
+
// socket). A timer that already fired is null and its sweep is tracked
|
|
619
|
+
// by connectInFlight, so startConnect below coalesces onto it instead of
|
|
620
|
+
// racing it.
|
|
621
|
+
if (reconnectTimer) {
|
|
622
|
+
clearTimeout(reconnectTimer);
|
|
623
|
+
reconnectTimer = null;
|
|
624
|
+
}
|
|
625
|
+
cleanup();
|
|
626
|
+
try {
|
|
627
|
+
oldSocket.close();
|
|
628
|
+
}
|
|
629
|
+
catch (_) { }
|
|
630
|
+
ws = null;
|
|
631
|
+
setStatus('closed');
|
|
632
|
+
// Re-establish on the next preferred (non-failed) node. If this attempt
|
|
633
|
+
// fails, fall back to the normal backoff.
|
|
634
|
+
startConnect().catch(() => scheduleReconnect());
|
|
635
|
+
}
|
|
543
636
|
function call(method, params, timeoutMs = rpcTimeoutMs) {
|
|
544
637
|
if (!ws || ws.readyState !== 1) {
|
|
545
638
|
return Promise.reject(new ConnectionError('WebSocket not open'));
|
|
@@ -593,6 +686,7 @@ function createTransport(config = {}) {
|
|
|
593
686
|
return {
|
|
594
687
|
connect,
|
|
595
688
|
disconnect,
|
|
689
|
+
forceReconnect,
|
|
596
690
|
call,
|
|
597
691
|
addMessageHandler,
|
|
598
692
|
getStatus,
|
|
@@ -41,6 +41,26 @@ import { withTimeout } from './order/utils/timeout.js';
|
|
|
41
41
|
* @returns {Function} Unregister function
|
|
42
42
|
*/
|
|
43
43
|
declare function onReconnect(callback: any): () => void;
|
|
44
|
+
/**
|
|
45
|
+
* Whether a transport-reported node failure should count as a persistent strike
|
|
46
|
+
* in the NodeManager ledger.
|
|
47
|
+
*
|
|
48
|
+
* Everything does EXCEPT 'forced-reconnect'. A forced reconnect is a
|
|
49
|
+
* session-level recovery attempt, not evidence the node is bad: the transport
|
|
50
|
+
* already deprioritizes that node in memory (failedNodes) for the next connect,
|
|
51
|
+
* and that deprioritization is self-clearing once any node connects. Strikes, by
|
|
52
|
+
* contrast, survive restarts and are only cleared by consecutive successful
|
|
53
|
+
* health probes 4h apart — so a false positive costs a healthy node 24h of the
|
|
54
|
+
* rotation. A node that only had a server-side login-session swap (the
|
|
55
|
+
* 2026-09-25 incident) is perfectly healthy, yet 3 forced reconnects 30s apart
|
|
56
|
+
* used to blacklist it.
|
|
57
|
+
*
|
|
58
|
+
* The fill-channel watchdog escalates to a real strike ('fill-channel-
|
|
59
|
+
* unrecoverable') once recovery has demonstrably failed, so genuinely bad nodes
|
|
60
|
+
* are still blacklisted — on the stronger evidence of repeated FAILED recovery
|
|
61
|
+
* rather than mere attempts.
|
|
62
|
+
*/
|
|
63
|
+
export declare function shouldCountNodeStrike(source?: string): boolean;
|
|
44
64
|
declare const _lazyBitShares: any;
|
|
45
65
|
/**
|
|
46
66
|
* Suppress or restore connection log output.
|
|
@@ -102,6 +102,28 @@ let _resolvers = null;
|
|
|
102
102
|
// connect attempt with the current generation and, on resolution, drop
|
|
103
103
|
// the result and force a disconnect if the generation has moved on.
|
|
104
104
|
let _connectGeneration = 0;
|
|
105
|
+
/**
|
|
106
|
+
* Whether a transport-reported node failure should count as a persistent strike
|
|
107
|
+
* in the NodeManager ledger.
|
|
108
|
+
*
|
|
109
|
+
* Everything does EXCEPT 'forced-reconnect'. A forced reconnect is a
|
|
110
|
+
* session-level recovery attempt, not evidence the node is bad: the transport
|
|
111
|
+
* already deprioritizes that node in memory (failedNodes) for the next connect,
|
|
112
|
+
* and that deprioritization is self-clearing once any node connects. Strikes, by
|
|
113
|
+
* contrast, survive restarts and are only cleared by consecutive successful
|
|
114
|
+
* health probes 4h apart — so a false positive costs a healthy node 24h of the
|
|
115
|
+
* rotation. A node that only had a server-side login-session swap (the
|
|
116
|
+
* 2026-09-25 incident) is perfectly healthy, yet 3 forced reconnects 30s apart
|
|
117
|
+
* used to blacklist it.
|
|
118
|
+
*
|
|
119
|
+
* The fill-channel watchdog escalates to a real strike ('fill-channel-
|
|
120
|
+
* unrecoverable') once recovery has demonstrably failed, so genuinely bad nodes
|
|
121
|
+
* are still blacklisted — on the stronger evidence of repeated FAILED recovery
|
|
122
|
+
* rather than mere attempts.
|
|
123
|
+
*/
|
|
124
|
+
export function shouldCountNodeStrike(source) {
|
|
125
|
+
return source !== 'forced-reconnect';
|
|
126
|
+
}
|
|
105
127
|
function ensureInitialized() {
|
|
106
128
|
if (_initialized)
|
|
107
129
|
return;
|
|
@@ -110,6 +132,13 @@ function ensureInitialized() {
|
|
|
110
132
|
// Single sink for every node-failure path (live transport + signing/builder
|
|
111
133
|
// fee fetch) so strike counting and blacklisting cannot diverge.
|
|
112
134
|
const reportNodeFailureToManager = (nodeUrl, errorMessage, source) => {
|
|
135
|
+
// A forced reconnect is a session-level RECOVERY ATTEMPT, not evidence
|
|
136
|
+
// that the node is bad — see shouldCountNodeStrike above. The transport's
|
|
137
|
+
// own in-memory deprioritization still applies, so the reconnect rotates
|
|
138
|
+
// away from a wedged node regardless; only the persistent strike is
|
|
139
|
+
// withheld, and the fill-channel watchdog issues it if recovery fails.
|
|
140
|
+
if (!shouldCountNodeStrike(source))
|
|
141
|
+
return;
|
|
113
142
|
if (nodeManager) {
|
|
114
143
|
nodeManager.reportNodeFailure(nodeUrl, errorMessage, source);
|
|
115
144
|
}
|
|
@@ -85,8 +85,17 @@ function validateBotEntry(b, i, src) {
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
if ('reserveOrders' in b && b.reserveOrders !== undefined && b.reserveOrders !== null) {
|
|
88
|
-
if (typeof b.reserveOrders
|
|
88
|
+
if (typeof b.reserveOrders === 'number') {
|
|
89
|
+
// Legacy numeric form migrates at read time (resolveReserveCount):
|
|
90
|
+
// { buy: n, sell: 0 }. Accept it here so a hand-edited bots.json
|
|
91
|
+
// that bypassed the editor is not rejected for a form the runtime
|
|
92
|
+
// still understands.
|
|
93
|
+
if (!Number.isInteger(b.reserveOrders) || b.reserveOrders < 0)
|
|
94
|
+
problems.push("'reserveOrders' numeric form must be a non-negative integer");
|
|
95
|
+
}
|
|
96
|
+
else if (typeof b.reserveOrders !== 'object' || Array.isArray(b.reserveOrders)) {
|
|
89
97
|
problems.push("'reserveOrders' must be an object {buy, sell}");
|
|
98
|
+
}
|
|
90
99
|
else {
|
|
91
100
|
for (const side of ['buy', 'sell']) {
|
|
92
101
|
const v = b.reserveOrders[side];
|