dexbot 1.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +287 -0
- package/claw/README.md +513 -0
- package/claw/scripts/memu_runner.py +385 -0
- package/claw/skills/bitshares-guide/SKILL.md +14 -0
- package/claw/skills/bitshares-guide/references/example-skill-shells.md +18 -0
- package/claw/skills/bitshares-guide/references/presentation-patterns.md +19 -0
- package/claw/skills/bitshares-guide/references/scope-guardrails.md +9 -0
- package/claw/skills/launcher-ops/SKILL.md +18 -0
- package/claw/skills/launcher-ops/references/launcher-workflow.md +23 -0
- package/claw/skills/margin-trading/SKILL.md +23 -0
- package/claw/skills/margin-trading/references/honest-asset-list.md +74 -0
- package/claw/skills/margin-trading/references/honest-assets.md +44 -0
- package/claw/skills/margin-trading/references/position-management.md +203 -0
- package/claw/skills/margin-trading/references/trading-concepts.md +80 -0
- package/claw/skills/memu-memory/SKILL.md +218 -0
- package/claw/skills/shared/references/js-automation-overview.md +24 -0
- package/claw/skills/shared/references/safety-and-staleness.md +11 -0
- package/claw/skills/shared/references/skill-boundaries.md +17 -0
- package/claw/skills/trend-detection/SKILL.md +33 -0
- package/claw/skills/trend-detection/agents/openai.yaml +3 -0
- package/claw/skills/trend-detection/references/service.md +45 -0
- package/dist/analysis/ama_fitting/analyze_ama_price_changes.d.ts +3 -0
- package/dist/analysis/ama_fitting/analyze_ama_price_changes.d.ts.map +1 -0
- package/dist/analysis/ama_fitting/analyze_ama_price_changes.js +149 -0
- package/dist/analysis/ama_fitting/analyze_ama_price_changes.js.map +1 -0
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.d.ts +3 -0
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.d.ts.map +1 -0
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +387 -0
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js.map +1 -0
- package/dist/analysis/ama_fitting/calibrate_convergence_er.d.ts +3 -0
- package/dist/analysis/ama_fitting/calibrate_convergence_er.d.ts.map +1 -0
- package/dist/analysis/ama_fitting/calibrate_convergence_er.js +200 -0
- package/dist/analysis/ama_fitting/calibrate_convergence_er.js.map +1 -0
- package/dist/analysis/ama_fitting/fetch_lp_candles.d.ts +3 -0
- package/dist/analysis/ama_fitting/fetch_lp_candles.d.ts.map +1 -0
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +212 -0
- package/dist/analysis/ama_fitting/fetch_lp_candles.js.map +1 -0
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +31 -0
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts.map +1 -0
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +249 -0
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js.map +1 -0
- package/dist/analysis/ama_fitting/optimizer_high_resolution.d.ts +2 -0
- package/dist/analysis/ama_fitting/optimizer_high_resolution.d.ts.map +1 -0
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js +782 -0
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js.map +1 -0
- package/dist/analysis/analyze_derivatives.d.ts +95 -0
- package/dist/analysis/analyze_derivatives.d.ts.map +1 -0
- package/dist/analysis/analyze_derivatives.js +295 -0
- package/dist/analysis/analyze_derivatives.js.map +1 -0
- package/dist/analysis/analyze_dynamic_weight.d.ts +14 -0
- package/dist/analysis/analyze_dynamic_weight.d.ts.map +1 -0
- package/dist/analysis/analyze_dynamic_weight.js +208 -0
- package/dist/analysis/analyze_dynamic_weight.js.map +1 -0
- package/dist/analysis/analyze_kalman.d.ts +13 -0
- package/dist/analysis/analyze_kalman.d.ts.map +1 -0
- package/dist/analysis/analyze_kalman.js +123 -0
- package/dist/analysis/analyze_kalman.js.map +1 -0
- package/dist/analysis/analyze_regime.d.ts +20 -0
- package/dist/analysis/analyze_regime.d.ts.map +1 -0
- package/dist/analysis/analyze_regime.js +139 -0
- package/dist/analysis/analyze_regime.js.map +1 -0
- package/dist/analysis/analyze_regime_windows.d.ts +19 -0
- package/dist/analysis/analyze_regime_windows.d.ts.map +1 -0
- package/dist/analysis/analyze_regime_windows.js +413 -0
- package/dist/analysis/analyze_regime_windows.js.map +1 -0
- package/dist/analysis/analyze_risk_profile.d.ts +3 -0
- package/dist/analysis/analyze_risk_profile.d.ts.map +1 -0
- package/dist/analysis/analyze_risk_profile.js +159 -0
- package/dist/analysis/analyze_risk_profile.js.map +1 -0
- package/dist/analysis/analyze_trade_heatmap.d.ts +3 -0
- package/dist/analysis/analyze_trade_heatmap.d.ts.map +1 -0
- package/dist/analysis/analyze_trade_heatmap.js +367 -0
- package/dist/analysis/analyze_trade_heatmap.js.map +1 -0
- package/dist/analysis/analyze_volatility.d.ts +22 -0
- package/dist/analysis/analyze_volatility.d.ts.map +1 -0
- package/dist/analysis/analyze_volatility.js +172 -0
- package/dist/analysis/analyze_volatility.js.map +1 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +99 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts.map +1 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js +804 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js.map +1 -0
- package/dist/analysis/bot_fitting/backtest_bot_fitting.d.ts +2 -0
- package/dist/analysis/bot_fitting/backtest_bot_fitting.d.ts.map +1 -0
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js +347 -0
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js.map +1 -0
- package/dist/analysis/bot_fitting/shared_utils.d.ts +15 -0
- package/dist/analysis/bot_fitting/shared_utils.d.ts.map +1 -0
- package/dist/analysis/bot_fitting/shared_utils.js +48 -0
- package/dist/analysis/bot_fitting/shared_utils.js.map +1 -0
- package/dist/analysis/bot_key_utils.d.ts +16 -0
- package/dist/analysis/bot_key_utils.d.ts.map +1 -0
- package/dist/analysis/bot_key_utils.js +61 -0
- package/dist/analysis/bot_key_utils.js.map +1 -0
- package/dist/analysis/bot_usage/discover_bot_accounts.d.ts +3 -0
- package/dist/analysis/bot_usage/discover_bot_accounts.d.ts.map +1 -0
- package/dist/analysis/bot_usage/discover_bot_accounts.js +498 -0
- package/dist/analysis/bot_usage/discover_bot_accounts.js.map +1 -0
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts +546 -0
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts.map +1 -0
- package/dist/analysis/bot_usage/kibana_bot_queries.js +427 -0
- package/dist/analysis/bot_usage/kibana_bot_queries.js.map +1 -0
- package/dist/analysis/chart_utils.d.ts +13 -0
- package/dist/analysis/chart_utils.d.ts.map +1 -0
- package/dist/analysis/chart_utils.js +142 -0
- package/dist/analysis/chart_utils.js.map +1 -0
- package/dist/analysis/derivative_chart_generator.d.ts +18 -0
- package/dist/analysis/derivative_chart_generator.d.ts.map +1 -0
- package/dist/analysis/derivative_chart_generator.js +896 -0
- package/dist/analysis/derivative_chart_generator.js.map +1 -0
- package/dist/analysis/math_utils.d.ts +35 -0
- package/dist/analysis/math_utils.d.ts.map +1 -0
- package/dist/analysis/math_utils.js +121 -0
- package/dist/analysis/math_utils.js.map +1 -0
- package/dist/analysis/price_sources.d.ts +34 -0
- package/dist/analysis/price_sources.d.ts.map +1 -0
- package/dist/analysis/price_sources.js +103 -0
- package/dist/analysis/price_sources.js.map +1 -0
- package/dist/analysis/trade_profitability.d.ts +115 -0
- package/dist/analysis/trade_profitability.d.ts.map +1 -0
- package/dist/analysis/trade_profitability.js +1247 -0
- package/dist/analysis/trade_profitability.js.map +1 -0
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +38 -0
- package/dist/analysis/tradingview/analyze_tradingview.d.ts.map +1 -0
- package/dist/analysis/tradingview/analyze_tradingview.js +212 -0
- package/dist/analysis/tradingview/analyze_tradingview.js.map +1 -0
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts +9 -0
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts.map +1 -0
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +1717 -0
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js.map +1 -0
- package/dist/analysis/trend_detection/derivative_analyzer.d.ts +250 -0
- package/dist/analysis/trend_detection/derivative_analyzer.d.ts.map +1 -0
- package/dist/analysis/trend_detection/derivative_analyzer.js +903 -0
- package/dist/analysis/trend_detection/derivative_analyzer.js.map +1 -0
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.d.ts +6 -0
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.d.ts.map +1 -0
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +1449 -0
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js.map +1 -0
- package/dist/analysis/trend_detection/hurst_analyzer.d.ts +42 -0
- package/dist/analysis/trend_detection/hurst_analyzer.d.ts.map +1 -0
- package/dist/analysis/trend_detection/hurst_analyzer.js +159 -0
- package/dist/analysis/trend_detection/hurst_analyzer.js.map +1 -0
- package/dist/analysis/trend_detection/kalman_chart_generator.d.ts +6 -0
- package/dist/analysis/trend_detection/kalman_chart_generator.d.ts.map +1 -0
- package/dist/analysis/trend_detection/kalman_chart_generator.js +408 -0
- package/dist/analysis/trend_detection/kalman_chart_generator.js.map +1 -0
- package/dist/analysis/trend_detection/kalman_trend_analyzer.d.ts +103 -0
- package/dist/analysis/trend_detection/kalman_trend_analyzer.d.ts.map +1 -0
- package/dist/analysis/trend_detection/kalman_trend_analyzer.js +229 -0
- package/dist/analysis/trend_detection/kalman_trend_analyzer.js.map +1 -0
- package/dist/analysis/trend_detection/kalman_velocity_smoothing.d.ts +34 -0
- package/dist/analysis/trend_detection/kalman_velocity_smoothing.d.ts.map +1 -0
- package/dist/analysis/trend_detection/kalman_velocity_smoothing.js +89 -0
- package/dist/analysis/trend_detection/kalman_velocity_smoothing.js.map +1 -0
- package/dist/analysis/trend_detection/permutation_entropy_analyzer.d.ts +39 -0
- package/dist/analysis/trend_detection/permutation_entropy_analyzer.d.ts.map +1 -0
- package/dist/analysis/trend_detection/permutation_entropy_analyzer.js +132 -0
- package/dist/analysis/trend_detection/permutation_entropy_analyzer.js.map +1 -0
- package/dist/analysis/trend_detection/regime_chart_generator.d.ts +6 -0
- package/dist/analysis/trend_detection/regime_chart_generator.d.ts.map +1 -0
- package/dist/analysis/trend_detection/regime_chart_generator.js +345 -0
- package/dist/analysis/trend_detection/regime_chart_generator.js.map +1 -0
- package/dist/analysis/trend_detection/tests/test_kalman_trend.d.ts +2 -0
- package/dist/analysis/trend_detection/tests/test_kalman_trend.d.ts.map +1 -0
- package/dist/analysis/trend_detection/tests/test_kalman_trend.js +130 -0
- package/dist/analysis/trend_detection/tests/test_kalman_trend.js.map +1 -0
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.d.ts +2 -0
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.d.ts.map +1 -0
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.js +38 -0
- package/dist/analysis/trend_detection/tests/test_kalman_velocity_smoothing.js.map +1 -0
- package/dist/analysis/trend_detection/volatility_chart_generator.d.ts +6 -0
- package/dist/analysis/trend_detection/volatility_chart_generator.d.ts.map +1 -0
- package/dist/analysis/trend_detection/volatility_chart_generator.js +732 -0
- package/dist/analysis/trend_detection/volatility_chart_generator.js.map +1 -0
- package/dist/bot.d.ts +49 -0
- package/dist/bot.d.ts.map +1 -0
- package/dist/bot.js +221 -0
- package/dist/bot.js.map +1 -0
- package/dist/credential-daemon.d.ts +64 -0
- package/dist/credential-daemon.d.ts.map +1 -0
- package/dist/credential-daemon.js +987 -0
- package/dist/credential-daemon.js.map +1 -0
- package/dist/dexbot.d.ts +87 -0
- package/dist/dexbot.d.ts.map +1 -0
- package/dist/dexbot.js +1222 -0
- package/dist/dexbot.js.map +1 -0
- package/dist/market_adapter/ama_signal_runner.d.ts +3 -0
- package/dist/market_adapter/ama_signal_runner.d.ts.map +1 -0
- package/dist/market_adapter/ama_signal_runner.js +175 -0
- package/dist/market_adapter/ama_signal_runner.js.map +1 -0
- package/dist/market_adapter/candle_utils.d.ts +57 -0
- package/dist/market_adapter/candle_utils.d.ts.map +1 -0
- package/dist/market_adapter/candle_utils.js +252 -0
- package/dist/market_adapter/candle_utils.js.map +1 -0
- package/dist/market_adapter/core/asymmetric_bounds.d.ts +20 -0
- package/dist/market_adapter/core/asymmetric_bounds.d.ts.map +1 -0
- package/dist/market_adapter/core/asymmetric_bounds.js +88 -0
- package/dist/market_adapter/core/asymmetric_bounds.js.map +1 -0
- package/dist/market_adapter/core/config_normalizers.d.ts +10 -0
- package/dist/market_adapter/core/config_normalizers.d.ts.map +1 -0
- package/dist/market_adapter/core/config_normalizers.js +24 -0
- package/dist/market_adapter/core/config_normalizers.js.map +1 -0
- package/dist/market_adapter/core/kibana_candles.d.ts +74 -0
- package/dist/market_adapter/core/kibana_candles.d.ts.map +1 -0
- package/dist/market_adapter/core/kibana_candles.js +249 -0
- package/dist/market_adapter/core/kibana_candles.js.map +1 -0
- package/dist/market_adapter/core/kibana_client.d.ts +27 -0
- package/dist/market_adapter/core/kibana_client.d.ts.map +1 -0
- package/dist/market_adapter/core/kibana_client.js +120 -0
- package/dist/market_adapter/core/kibana_client.js.map +1 -0
- package/dist/market_adapter/core/kibana_market_candles.d.ts +143 -0
- package/dist/market_adapter/core/kibana_market_candles.d.ts.map +1 -0
- package/dist/market_adapter/core/kibana_market_candles.js +170 -0
- package/dist/market_adapter/core/kibana_market_candles.js.map +1 -0
- package/dist/market_adapter/core/market_adapter_service.d.ts +210 -0
- package/dist/market_adapter/core/market_adapter_service.d.ts.map +1 -0
- package/dist/market_adapter/core/market_adapter_service.js +2328 -0
- package/dist/market_adapter/core/market_adapter_service.js.map +1 -0
- package/dist/market_adapter/core/strategies/ama.d.ts +2 -0
- package/dist/market_adapter/core/strategies/ama.d.ts.map +1 -0
- package/dist/market_adapter/core/strategies/ama.js +138 -0
- package/dist/market_adapter/core/strategies/ama.js.map +1 -0
- package/dist/market_adapter/core/strategies/ama_slope_model.d.ts +50 -0
- package/dist/market_adapter/core/strategies/ama_slope_model.d.ts.map +1 -0
- package/dist/market_adapter/core/strategies/ama_slope_model.js +143 -0
- package/dist/market_adapter/core/strategies/ama_slope_model.js.map +1 -0
- package/dist/market_adapter/core/strategies/atr/calculator.d.ts +12 -0
- package/dist/market_adapter/core/strategies/atr/calculator.d.ts.map +1 -0
- package/dist/market_adapter/core/strategies/atr/calculator.js +47 -0
- package/dist/market_adapter/core/strategies/atr/calculator.js.map +1 -0
- package/dist/market_adapter/core/strategies/collateral_manager.d.ts +32 -0
- package/dist/market_adapter/core/strategies/collateral_manager.d.ts.map +1 -0
- package/dist/market_adapter/core/strategies/collateral_manager.js +68 -0
- package/dist/market_adapter/core/strategies/collateral_manager.js.map +1 -0
- package/dist/market_adapter/core/strategies/regime_gate.d.ts +66 -0
- package/dist/market_adapter/core/strategies/regime_gate.d.ts.map +1 -0
- package/dist/market_adapter/core/strategies/regime_gate.js +189 -0
- package/dist/market_adapter/core/strategies/regime_gate.js.map +1 -0
- package/dist/market_adapter/index.d.ts +10 -0
- package/dist/market_adapter/index.d.ts.map +1 -0
- package/dist/market_adapter/index.js +71 -0
- package/dist/market_adapter/index.js.map +1 -0
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.d.ts +3 -0
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.d.ts.map +1 -0
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +1165 -0
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js.map +1 -0
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +54 -0
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts.map +1 -0
- package/dist/market_adapter/inputs/fetch_lp_data.js +743 -0
- package/dist/market_adapter/inputs/fetch_lp_data.js.map +1 -0
- package/dist/market_adapter/inputs/kibana_source.d.ts +68 -0
- package/dist/market_adapter/inputs/kibana_source.d.ts.map +1 -0
- package/dist/market_adapter/inputs/kibana_source.js +143 -0
- package/dist/market_adapter/inputs/kibana_source.js.map +1 -0
- package/dist/market_adapter/interval_utils.d.ts +6 -0
- package/dist/market_adapter/interval_utils.d.ts.map +1 -0
- package/dist/market_adapter/interval_utils.js +14 -0
- package/dist/market_adapter/interval_utils.js.map +1 -0
- package/dist/market_adapter/log_format.d.ts +16 -0
- package/dist/market_adapter/log_format.d.ts.map +1 -0
- package/dist/market_adapter/log_format.js +72 -0
- package/dist/market_adapter/log_format.js.map +1 -0
- package/dist/market_adapter/lp_chart_core.d.ts +6 -0
- package/dist/market_adapter/lp_chart_core.d.ts.map +1 -0
- package/dist/market_adapter/lp_chart_core.js +874 -0
- package/dist/market_adapter/lp_chart_core.js.map +1 -0
- package/dist/market_adapter/lp_chart_runner.d.ts +135 -0
- package/dist/market_adapter/lp_chart_runner.d.ts.map +1 -0
- package/dist/market_adapter/lp_chart_runner.js +337 -0
- package/dist/market_adapter/lp_chart_runner.js.map +1 -0
- package/dist/market_adapter/lp_chart_strategy_loader.d.ts +23 -0
- package/dist/market_adapter/lp_chart_strategy_loader.d.ts.map +1 -0
- package/dist/market_adapter/lp_chart_strategy_loader.js +169 -0
- package/dist/market_adapter/lp_chart_strategy_loader.js.map +1 -0
- package/dist/market_adapter/market_adapter.d.ts +133 -0
- package/dist/market_adapter/market_adapter.d.ts.map +1 -0
- package/dist/market_adapter/market_adapter.js +1331 -0
- package/dist/market_adapter/market_adapter.js.map +1 -0
- package/dist/market_adapter/merge_lp_data.d.ts +3 -0
- package/dist/market_adapter/merge_lp_data.d.ts.map +1 -0
- package/dist/market_adapter/merge_lp_data.js +125 -0
- package/dist/market_adapter/merge_lp_data.js.map +1 -0
- package/dist/market_adapter/test_helpers.d.ts +14 -0
- package/dist/market_adapter/test_helpers.d.ts.map +1 -0
- package/dist/market_adapter/test_helpers.js +22 -0
- package/dist/market_adapter/test_helpers.js.map +1 -0
- package/dist/market_adapter/utils/adapter_client.d.ts +20 -0
- package/dist/market_adapter/utils/adapter_client.d.ts.map +1 -0
- package/dist/market_adapter/utils/adapter_client.js +88 -0
- package/dist/market_adapter/utils/adapter_client.js.map +1 -0
- package/dist/market_adapter/utils/atomic_write.d.ts +9 -0
- package/dist/market_adapter/utils/atomic_write.d.ts.map +1 -0
- package/dist/market_adapter/utils/atomic_write.js +13 -0
- package/dist/market_adapter/utils/atomic_write.js.map +1 -0
- package/dist/market_adapter/utils/chain.d.ts +50 -0
- package/dist/market_adapter/utils/chain.d.ts.map +1 -0
- package/dist/market_adapter/utils/chain.js +196 -0
- package/dist/market_adapter/utils/chain.js.map +1 -0
- package/dist/market_adapter/utils/data_discovery.d.ts +6 -0
- package/dist/market_adapter/utils/data_discovery.d.ts.map +1 -0
- package/dist/market_adapter/utils/data_discovery.js +35 -0
- package/dist/market_adapter/utils/data_discovery.js.map +1 -0
- package/dist/market_adapter/utils/dynamic_grid_snapshot.d.ts +15 -0
- package/dist/market_adapter/utils/dynamic_grid_snapshot.d.ts.map +1 -0
- package/dist/market_adapter/utils/dynamic_grid_snapshot.js +60 -0
- package/dist/market_adapter/utils/dynamic_grid_snapshot.js.map +1 -0
- package/dist/market_adapter/utils/file_lock.d.ts +19 -0
- package/dist/market_adapter/utils/file_lock.d.ts.map +1 -0
- package/dist/market_adapter/utils/file_lock.js +191 -0
- package/dist/market_adapter/utils/file_lock.js.map +1 -0
- package/dist/market_adapter/utils/native_history.d.ts +19 -0
- package/dist/market_adapter/utils/native_history.d.ts.map +1 -0
- package/dist/market_adapter/utils/native_history.js +172 -0
- package/dist/market_adapter/utils/native_history.js.map +1 -0
- package/dist/market_adapter/utils/paths.d.ts +5 -0
- package/dist/market_adapter/utils/paths.d.ts.map +1 -0
- package/dist/market_adapter/utils/paths.js +7 -0
- package/dist/market_adapter/utils/paths.js.map +1 -0
- package/dist/modules/account_bots.d.ts +112 -0
- package/dist/modules/account_bots.d.ts.map +1 -0
- package/dist/modules/account_bots.js +1259 -0
- package/dist/modules/account_bots.js.map +1 -0
- package/dist/modules/account_orders.d.ts +273 -0
- package/dist/modules/account_orders.d.ts.map +1 -0
- package/dist/modules/account_orders.js +662 -0
- package/dist/modules/account_orders.js.map +1 -0
- package/dist/modules/authority_resolver.d.ts +2 -0
- package/dist/modules/authority_resolver.d.ts.map +1 -0
- package/dist/modules/authority_resolver.js +231 -0
- package/dist/modules/authority_resolver.js.map +1 -0
- package/dist/modules/bitshares-native/chain_client.d.ts +38 -0
- package/dist/modules/bitshares-native/chain_client.d.ts.map +1 -0
- package/dist/modules/bitshares-native/chain_client.js +322 -0
- package/dist/modules/bitshares-native/chain_client.js.map +1 -0
- package/dist/modules/bitshares-native/crypto/ecc.browser.d.ts +59 -0
- package/dist/modules/bitshares-native/crypto/ecc.browser.d.ts.map +1 -0
- package/dist/modules/bitshares-native/crypto/ecc.browser.js +456 -0
- package/dist/modules/bitshares-native/crypto/ecc.browser.js.map +1 -0
- package/dist/modules/bitshares-native/crypto/ecc.d.ts +66 -0
- package/dist/modules/bitshares-native/crypto/ecc.d.ts.map +1 -0
- package/dist/modules/bitshares-native/crypto/ecc.js +557 -0
- package/dist/modules/bitshares-native/crypto/ecc.js.map +1 -0
- package/dist/modules/bitshares-native/crypto/ecc_selector.d.ts +2 -0
- package/dist/modules/bitshares-native/crypto/ecc_selector.d.ts.map +1 -0
- package/dist/modules/bitshares-native/crypto/ecc_selector.js +20 -0
- package/dist/modules/bitshares-native/crypto/ecc_selector.js.map +1 -0
- package/dist/modules/bitshares-native/index.d.ts +3 -0
- package/dist/modules/bitshares-native/index.d.ts.map +1 -0
- package/dist/modules/bitshares-native/index.js +32 -0
- package/dist/modules/bitshares-native/index.js.map +1 -0
- package/dist/modules/bitshares-native/resolvers.d.ts +40 -0
- package/dist/modules/bitshares-native/resolvers.d.ts.map +1 -0
- package/dist/modules/bitshares-native/resolvers.js +161 -0
- package/dist/modules/bitshares-native/resolvers.js.map +1 -0
- package/dist/modules/bitshares-native/serial/chain_constants.d.ts +56 -0
- package/dist/modules/bitshares-native/serial/chain_constants.d.ts.map +1 -0
- package/dist/modules/bitshares-native/serial/chain_constants.js +92 -0
- package/dist/modules/bitshares-native/serial/chain_constants.js.map +1 -0
- package/dist/modules/bitshares-native/serial/index.d.ts +10 -0
- package/dist/modules/bitshares-native/serial/index.d.ts.map +1 -0
- package/dist/modules/bitshares-native/serial/index.js +14 -0
- package/dist/modules/bitshares-native/serial/index.js.map +1 -0
- package/dist/modules/bitshares-native/serial/operations.d.ts +102 -0
- package/dist/modules/bitshares-native/serial/operations.d.ts.map +1 -0
- package/dist/modules/bitshares-native/serial/operations.js +615 -0
- package/dist/modules/bitshares-native/serial/operations.js.map +1 -0
- package/dist/modules/bitshares-native/serial/serializer.d.ts +64 -0
- package/dist/modules/bitshares-native/serial/serializer.d.ts.map +1 -0
- package/dist/modules/bitshares-native/serial/serializer.js +252 -0
- package/dist/modules/bitshares-native/serial/serializer.js.map +1 -0
- package/dist/modules/bitshares-native/serial/types.d.ts +120 -0
- package/dist/modules/bitshares-native/serial/types.d.ts.map +1 -0
- package/dist/modules/bitshares-native/serial/types.js +837 -0
- package/dist/modules/bitshares-native/serial/types.js.map +1 -0
- package/dist/modules/bitshares-native/signing_client.d.ts +6 -0
- package/dist/modules/bitshares-native/signing_client.d.ts.map +1 -0
- package/dist/modules/bitshares-native/signing_client.js +141 -0
- package/dist/modules/bitshares-native/signing_client.js.map +1 -0
- package/dist/modules/bitshares-native/subscriptions.d.ts +6 -0
- package/dist/modules/bitshares-native/subscriptions.d.ts.map +1 -0
- package/dist/modules/bitshares-native/subscriptions.js +865 -0
- package/dist/modules/bitshares-native/subscriptions.js.map +1 -0
- package/dist/modules/bitshares-native/transport.d.ts +51 -0
- package/dist/modules/bitshares-native/transport.d.ts.map +1 -0
- package/dist/modules/bitshares-native/transport.js +447 -0
- package/dist/modules/bitshares-native/transport.js.map +1 -0
- package/dist/modules/bitshares-native/tx/builder.d.ts +30 -0
- package/dist/modules/bitshares-native/tx/builder.d.ts.map +1 -0
- package/dist/modules/bitshares-native/tx/builder.js +262 -0
- package/dist/modules/bitshares-native/tx/builder.js.map +1 -0
- package/dist/modules/bitshares_client.d.ts +129 -0
- package/dist/modules/bitshares_client.d.ts.map +1 -0
- package/dist/modules/bitshares_client.js +627 -0
- package/dist/modules/bitshares_client.js.map +1 -0
- package/dist/modules/bot_settings.d.ts +33 -0
- package/dist/modules/bot_settings.d.ts.map +1 -0
- package/dist/modules/bot_settings.js +328 -0
- package/dist/modules/bot_settings.js.map +1 -0
- package/dist/modules/bots_file_lock.d.ts +88 -0
- package/dist/modules/bots_file_lock.d.ts.map +1 -0
- package/dist/modules/bots_file_lock.js +161 -0
- package/dist/modules/bots_file_lock.js.map +1 -0
- package/dist/modules/chain_keys.d.ts +302 -0
- package/dist/modules/chain_keys.d.ts.map +1 -0
- package/dist/modules/chain_keys.js +1095 -0
- package/dist/modules/chain_keys.js.map +1 -0
- package/dist/modules/chain_orders.d.ts +423 -0
- package/dist/modules/chain_orders.d.ts.map +1 -0
- package/dist/modules/chain_orders.js +1201 -0
- package/dist/modules/chain_orders.js.map +1 -0
- package/dist/modules/cli_whitelist_args.d.ts +6 -0
- package/dist/modules/cli_whitelist_args.d.ts.map +1 -0
- package/dist/modules/cli_whitelist_args.js +13 -0
- package/dist/modules/cli_whitelist_args.js.map +1 -0
- package/dist/modules/config.d.ts +57 -0
- package/dist/modules/config.d.ts.map +1 -0
- package/dist/modules/config.js +107 -0
- package/dist/modules/config.js.map +1 -0
- package/dist/modules/constants.d.ts +635 -0
- package/dist/modules/constants.d.ts.map +1 -0
- package/dist/modules/constants.js +1464 -0
- package/dist/modules/constants.js.map +1 -0
- package/dist/modules/cr_planner.d.ts +51 -0
- package/dist/modules/cr_planner.d.ts.map +1 -0
- package/dist/modules/cr_planner.js +299 -0
- package/dist/modules/cr_planner.js.map +1 -0
- package/dist/modules/credential_policy.d.ts +188 -0
- package/dist/modules/credential_policy.d.ts.map +1 -0
- package/dist/modules/credential_policy.js +1452 -0
- package/dist/modules/credential_policy.js.map +1 -0
- package/dist/modules/credential_runtime.d.ts +42 -0
- package/dist/modules/credential_runtime.d.ts.map +1 -0
- package/dist/modules/credential_runtime.js +148 -0
- package/dist/modules/credential_runtime.js.map +1 -0
- package/dist/modules/credential_session_cache.d.ts +36 -0
- package/dist/modules/credential_session_cache.d.ts.map +1 -0
- package/dist/modules/credential_session_cache.js +106 -0
- package/dist/modules/credential_session_cache.js.map +1 -0
- package/dist/modules/credit_runtime.d.ts +412 -0
- package/dist/modules/credit_runtime.d.ts.map +1 -0
- package/dist/modules/credit_runtime.js +2866 -0
- package/dist/modules/credit_runtime.js.map +1 -0
- package/dist/modules/crypto/browser_provider.d.ts +15 -0
- package/dist/modules/crypto/browser_provider.d.ts.map +1 -0
- package/dist/modules/crypto/browser_provider.js +113 -0
- package/dist/modules/crypto/browser_provider.js.map +1 -0
- package/dist/modules/crypto/index.d.ts +9 -0
- package/dist/modules/crypto/index.d.ts.map +1 -0
- package/dist/modules/crypto/index.js +44 -0
- package/dist/modules/crypto/index.js.map +1 -0
- package/dist/modules/crypto/node_provider.d.ts +15 -0
- package/dist/modules/crypto/node_provider.d.ts.map +1 -0
- package/dist/modules/crypto/node_provider.js +74 -0
- package/dist/modules/crypto/node_provider.js.map +1 -0
- package/dist/modules/crypto/provider.d.ts +34 -0
- package/dist/modules/crypto/provider.d.ts.map +1 -0
- package/dist/modules/crypto/provider.js +3 -0
- package/dist/modules/crypto/provider.js.map +1 -0
- package/dist/modules/crypto/pure_ripemd160.d.ts +8 -0
- package/dist/modules/crypto/pure_ripemd160.d.ts.map +1 -0
- package/dist/modules/crypto/pure_ripemd160.js +142 -0
- package/dist/modules/crypto/pure_ripemd160.js.map +1 -0
- package/dist/modules/crypto/pure_scrypt.d.ts +23 -0
- package/dist/modules/crypto/pure_scrypt.d.ts.map +1 -0
- package/dist/modules/crypto/pure_scrypt.js +161 -0
- package/dist/modules/crypto/pure_scrypt.js.map +1 -0
- package/dist/modules/crypto/pure_secp256k1.d.ts +29 -0
- package/dist/modules/crypto/pure_secp256k1.d.ts.map +1 -0
- package/dist/modules/crypto/pure_secp256k1.js +195 -0
- package/dist/modules/crypto/pure_secp256k1.js.map +1 -0
- package/dist/modules/crypto/sync.d.ts +13 -0
- package/dist/modules/crypto/sync.d.ts.map +1 -0
- package/dist/modules/crypto/sync.js +37 -0
- package/dist/modules/crypto/sync.js.map +1 -0
- package/dist/modules/dexbot_class.d.ts +1074 -0
- package/dist/modules/dexbot_class.d.ts.map +1 -0
- package/dist/modules/dexbot_class.js +4613 -0
- package/dist/modules/dexbot_class.js.map +1 -0
- package/dist/modules/dexbot_credential_client.d.ts +71 -0
- package/dist/modules/dexbot_credential_client.d.ts.map +1 -0
- package/dist/modules/dexbot_credential_client.js +224 -0
- package/dist/modules/dexbot_credential_client.js.map +1 -0
- package/dist/modules/dexbot_fill_runtime.d.ts +173 -0
- package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -0
- package/dist/modules/dexbot_fill_runtime.js +217 -0
- package/dist/modules/dexbot_fill_runtime.js.map +1 -0
- package/dist/modules/dexbot_maintenance_runtime.d.ts +279 -0
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -0
- package/dist/modules/dexbot_maintenance_runtime.js +1613 -0
- package/dist/modules/dexbot_maintenance_runtime.js.map +1 -0
- package/dist/modules/env.d.ts +12 -0
- package/dist/modules/env.d.ts.map +1 -0
- package/dist/modules/env.js +57 -0
- package/dist/modules/env.js.map +1 -0
- package/dist/modules/fund_registry.d.ts +96 -0
- package/dist/modules/fund_registry.d.ts.map +1 -0
- package/dist/modules/fund_registry.js +261 -0
- package/dist/modules/fund_registry.js.map +1 -0
- package/dist/modules/general_settings.d.ts +36 -0
- package/dist/modules/general_settings.d.ts.map +1 -0
- package/dist/modules/general_settings.js +63 -0
- package/dist/modules/general_settings.js.map +1 -0
- package/dist/modules/graceful_shutdown.d.ts +86 -0
- package/dist/modules/graceful_shutdown.d.ts.map +1 -0
- package/dist/modules/graceful_shutdown.js +164 -0
- package/dist/modules/graceful_shutdown.js.map +1 -0
- package/dist/modules/key_store.d.ts +60 -0
- package/dist/modules/key_store.d.ts.map +1 -0
- package/dist/modules/key_store.js +215 -0
- package/dist/modules/key_store.js.map +1 -0
- package/dist/modules/launcher/bot_supervisor.d.ts +84 -0
- package/dist/modules/launcher/bot_supervisor.d.ts.map +1 -0
- package/dist/modules/launcher/bot_supervisor.js +1067 -0
- package/dist/modules/launcher/bot_supervisor.js.map +1 -0
- package/dist/modules/launcher/child_env.d.ts +9 -0
- package/dist/modules/launcher/child_env.d.ts.map +1 -0
- package/dist/modules/launcher/child_env.js +78 -0
- package/dist/modules/launcher/child_env.js.map +1 -0
- package/dist/modules/launcher/credential_bootstrap.d.ts +22 -0
- package/dist/modules/launcher/credential_bootstrap.d.ts.map +1 -0
- package/dist/modules/launcher/credential_bootstrap.js +303 -0
- package/dist/modules/launcher/credential_bootstrap.js.map +1 -0
- package/dist/modules/launcher/credential_daemon.d.ts +32 -0
- package/dist/modules/launcher/credential_daemon.d.ts.map +1 -0
- package/dist/modules/launcher/credential_daemon.js +173 -0
- package/dist/modules/launcher/credential_daemon.js.map +1 -0
- package/dist/modules/launcher/credential_secret.d.ts +6 -0
- package/dist/modules/launcher/credential_secret.d.ts.map +1 -0
- package/dist/modules/launcher/credential_secret.js +15 -0
- package/dist/modules/launcher/credential_secret.js.map +1 -0
- package/dist/modules/launcher/foreign_cred_daemon.d.ts +2 -0
- package/dist/modules/launcher/foreign_cred_daemon.d.ts.map +1 -0
- package/dist/modules/launcher/foreign_cred_daemon.js +166 -0
- package/dist/modules/launcher/foreign_cred_daemon.js.map +1 -0
- package/dist/modules/launcher/headless_password.d.ts +8 -0
- package/dist/modules/launcher/headless_password.d.ts.map +1 -0
- package/dist/modules/launcher/headless_password.js +33 -0
- package/dist/modules/launcher/headless_password.js.map +1 -0
- package/dist/modules/launcher/launch_modes.d.ts +36 -0
- package/dist/modules/launcher/launch_modes.d.ts.map +1 -0
- package/dist/modules/launcher/launch_modes.js +129 -0
- package/dist/modules/launcher/launch_modes.js.map +1 -0
- package/dist/modules/launcher/market_adapter_runtime.d.ts +43 -0
- package/dist/modules/launcher/market_adapter_runtime.d.ts.map +1 -0
- package/dist/modules/launcher/market_adapter_runtime.js +217 -0
- package/dist/modules/launcher/market_adapter_runtime.js.map +1 -0
- package/dist/modules/launcher/market_adapter_watchdog.d.ts +26 -0
- package/dist/modules/launcher/market_adapter_watchdog.d.ts.map +1 -0
- package/dist/modules/launcher/market_adapter_watchdog.js +214 -0
- package/dist/modules/launcher/market_adapter_watchdog.js.map +1 -0
- package/dist/modules/launcher/monolithic_runtime.d.ts +89 -0
- package/dist/modules/launcher/monolithic_runtime.d.ts.map +1 -0
- package/dist/modules/launcher/monolithic_runtime.js +425 -0
- package/dist/modules/launcher/monolithic_runtime.js.map +1 -0
- package/dist/modules/launcher/runtime_entry.d.ts +26 -0
- package/dist/modules/launcher/runtime_entry.d.ts.map +1 -0
- package/dist/modules/launcher/runtime_entry.js +49 -0
- package/dist/modules/launcher/runtime_entry.js.map +1 -0
- package/dist/modules/launcher/status_reporting.d.ts +48 -0
- package/dist/modules/launcher/status_reporting.d.ts.map +1 -0
- package/dist/modules/launcher/status_reporting.js +113 -0
- package/dist/modules/launcher/status_reporting.js.map +1 -0
- package/dist/modules/launcher/supervisor_control.d.ts +6 -0
- package/dist/modules/launcher/supervisor_control.d.ts.map +1 -0
- package/dist/modules/launcher/supervisor_control.js +59 -0
- package/dist/modules/launcher/supervisor_control.js.map +1 -0
- package/dist/modules/logger.d.ts +2 -0
- package/dist/modules/logger.d.ts.map +1 -0
- package/dist/modules/logger.js +71 -0
- package/dist/modules/logger.js.map +1 -0
- package/dist/modules/market_adapter_whitelist.d.ts +22 -0
- package/dist/modules/market_adapter_whitelist.d.ts.map +1 -0
- package/dist/modules/market_adapter_whitelist.js +82 -0
- package/dist/modules/market_adapter_whitelist.js.map +1 -0
- package/dist/modules/node_health_cache.d.ts +55 -0
- package/dist/modules/node_health_cache.d.ts.map +1 -0
- package/dist/modules/node_health_cache.js +134 -0
- package/dist/modules/node_health_cache.js.map +1 -0
- package/dist/modules/node_manager.d.ts +215 -0
- package/dist/modules/node_manager.d.ts.map +1 -0
- package/dist/modules/node_manager.js +628 -0
- package/dist/modules/node_manager.js.map +1 -0
- package/dist/modules/order/accounting.d.ts +349 -0
- package/dist/modules/order/accounting.d.ts.map +1 -0
- package/dist/modules/order/accounting.js +1191 -0
- package/dist/modules/order/accounting.js.map +1 -0
- package/dist/modules/order/async_lock.d.ts +126 -0
- package/dist/modules/order/async_lock.d.ts.map +1 -0
- package/dist/modules/order/async_lock.js +195 -0
- package/dist/modules/order/async_lock.js.map +1 -0
- package/dist/modules/order/export.d.ts +129 -0
- package/dist/modules/order/export.d.ts.map +1 -0
- package/dist/modules/order/export.js +316 -0
- package/dist/modules/order/export.js.map +1 -0
- package/dist/modules/order/format.d.ts +158 -0
- package/dist/modules/order/format.d.ts.map +1 -0
- package/dist/modules/order/format.js +232 -0
- package/dist/modules/order/format.js.map +1 -0
- package/dist/modules/order/grid.d.ts +549 -0
- package/dist/modules/order/grid.d.ts.map +1 -0
- package/dist/modules/order/grid.js +1746 -0
- package/dist/modules/order/grid.js.map +1 -0
- package/dist/modules/order/grid_reconcile.d.ts +154 -0
- package/dist/modules/order/grid_reconcile.d.ts.map +1 -0
- package/dist/modules/order/grid_reconcile.js +1192 -0
- package/dist/modules/order/grid_reconcile.js.map +1 -0
- package/dist/modules/order/index.d.ts +55 -0
- package/dist/modules/order/index.d.ts.map +1 -0
- package/dist/modules/order/index.js +79 -0
- package/dist/modules/order/index.js.map +1 -0
- package/dist/modules/order/logger.d.ts +136 -0
- package/dist/modules/order/logger.d.ts.map +1 -0
- package/dist/modules/order/logger.js +614 -0
- package/dist/modules/order/logger.js.map +1 -0
- package/dist/modules/order/logger_state.d.ts +133 -0
- package/dist/modules/order/logger_state.d.ts.map +1 -0
- package/dist/modules/order/logger_state.js +170 -0
- package/dist/modules/order/logger_state.js.map +1 -0
- package/dist/modules/order/manager.d.ts +413 -0
- package/dist/modules/order/manager.d.ts.map +1 -0
- package/dist/modules/order/manager.js +1605 -0
- package/dist/modules/order/manager.js.map +1 -0
- package/dist/modules/order/processed_fill_store.d.ts +135 -0
- package/dist/modules/order/processed_fill_store.d.ts.map +1 -0
- package/dist/modules/order/processed_fill_store.js +288 -0
- package/dist/modules/order/processed_fill_store.js.map +1 -0
- package/dist/modules/order/runner.d.ts +70 -0
- package/dist/modules/order/runner.d.ts.map +1 -0
- package/dist/modules/order/runner.js +140 -0
- package/dist/modules/order/runner.js.map +1 -0
- package/dist/modules/order/strategy.d.ts +196 -0
- package/dist/modules/order/strategy.d.ts.map +1 -0
- package/dist/modules/order/strategy.js +398 -0
- package/dist/modules/order/strategy.js.map +1 -0
- package/dist/modules/order/sync_engine.d.ts +386 -0
- package/dist/modules/order/sync_engine.d.ts.map +1 -0
- package/dist/modules/order/sync_engine.js +1459 -0
- package/dist/modules/order/sync_engine.js.map +1 -0
- package/dist/modules/order/utils/math.d.ts +677 -0
- package/dist/modules/order/utils/math.d.ts.map +1 -0
- package/dist/modules/order/utils/math.js +1172 -0
- package/dist/modules/order/utils/math.js.map +1 -0
- package/dist/modules/order/utils/order.d.ts +572 -0
- package/dist/modules/order/utils/order.d.ts.map +1 -0
- package/dist/modules/order/utils/order.js +1145 -0
- package/dist/modules/order/utils/order.js.map +1 -0
- package/dist/modules/order/utils/system.d.ts +268 -0
- package/dist/modules/order/utils/system.d.ts.map +1 -0
- package/dist/modules/order/utils/system.js +1261 -0
- package/dist/modules/order/utils/system.js.map +1 -0
- package/dist/modules/order/utils/validate.d.ts +280 -0
- package/dist/modules/order/utils/validate.d.ts.map +1 -0
- package/dist/modules/order/utils/validate.js +918 -0
- package/dist/modules/order/utils/validate.js.map +1 -0
- package/dist/modules/order/working_grid.d.ts +212 -0
- package/dist/modules/order/working_grid.d.ts.map +1 -0
- package/dist/modules/order/working_grid.js +279 -0
- package/dist/modules/order/working_grid.js.map +1 -0
- package/dist/modules/path_api.d.ts +32 -0
- package/dist/modules/path_api.d.ts.map +1 -0
- package/dist/modules/path_api.js +148 -0
- package/dist/modules/path_api.js.map +1 -0
- package/dist/modules/paths.d.ts +58 -0
- package/dist/modules/paths.d.ts.map +1 -0
- package/dist/modules/paths.js +82 -0
- package/dist/modules/paths.js.map +1 -0
- package/dist/modules/process_discovery.d.ts +58 -0
- package/dist/modules/process_discovery.d.ts.map +1 -0
- package/dist/modules/process_discovery.js +265 -0
- package/dist/modules/process_discovery.js.map +1 -0
- package/dist/modules/runtime.d.ts +34 -0
- package/dist/modules/runtime.d.ts.map +1 -0
- package/dist/modules/runtime.js +98 -0
- package/dist/modules/runtime.js.map +1 -0
- package/dist/modules/runtime_settings.d.ts +12 -0
- package/dist/modules/runtime_settings.d.ts.map +1 -0
- package/dist/modules/runtime_settings.js +122 -0
- package/dist/modules/runtime_settings.js.map +1 -0
- package/dist/modules/settings_merge.d.ts +38 -0
- package/dist/modules/settings_merge.d.ts.map +1 -0
- package/dist/modules/settings_merge.js +246 -0
- package/dist/modules/settings_merge.js.map +1 -0
- package/dist/modules/storage/browser_adapter.d.ts +56 -0
- package/dist/modules/storage/browser_adapter.d.ts.map +1 -0
- package/dist/modules/storage/browser_adapter.js +231 -0
- package/dist/modules/storage/browser_adapter.js.map +1 -0
- package/dist/modules/storage/index.d.ts +28 -0
- package/dist/modules/storage/index.d.ts.map +1 -0
- package/dist/modules/storage/index.js +52 -0
- package/dist/modules/storage/index.js.map +1 -0
- package/dist/modules/storage/node_adapter.d.ts +49 -0
- package/dist/modules/storage/node_adapter.d.ts.map +1 -0
- package/dist/modules/storage/node_adapter.js +167 -0
- package/dist/modules/storage/node_adapter.js.map +1 -0
- package/dist/modules/storage/types.d.ts +92 -0
- package/dist/modules/storage/types.d.ts.map +1 -0
- package/dist/modules/storage/types.js +3 -0
- package/dist/modules/storage/types.js.map +1 -0
- package/dist/modules/types.d.ts +1382 -0
- package/dist/modules/types.d.ts.map +1 -0
- package/dist/modules/types.js +11 -0
- package/dist/modules/types.js.map +1 -0
- package/dist/modules/utils/base58check.d.ts +12 -0
- package/dist/modules/utils/base58check.d.ts.map +1 -0
- package/dist/modules/utils/base58check.js +112 -0
- package/dist/modules/utils/base58check.js.map +1 -0
- package/dist/modules/utils/build_dir.d.ts +7 -0
- package/dist/modules/utils/build_dir.d.ts.map +1 -0
- package/dist/modules/utils/build_dir.js +8 -0
- package/dist/modules/utils/build_dir.js.map +1 -0
- package/dist/modules/utils/fs_utils.d.ts +25 -0
- package/dist/modules/utils/fs_utils.d.ts.map +1 -0
- package/dist/modules/utils/fs_utils.js +27 -0
- package/dist/modules/utils/fs_utils.js.map +1 -0
- package/dist/modules/utils/math_utils.d.ts +10 -0
- package/dist/modules/utils/math_utils.d.ts.map +1 -0
- package/dist/modules/utils/math_utils.js +17 -0
- package/dist/modules/utils/math_utils.js.map +1 -0
- package/dist/modules/validate_profiles.d.ts +37 -0
- package/dist/modules/validate_profiles.d.ts.map +1 -0
- package/dist/modules/validate_profiles.js +369 -0
- package/dist/modules/validate_profiles.js.map +1 -0
- package/dist/pm2.d.ts +167 -0
- package/dist/pm2.d.ts.map +1 -0
- package/dist/pm2.js +998 -0
- package/dist/pm2.js.map +1 -0
- package/dist/scripts/analyze-git.d.ts +11 -0
- package/dist/scripts/analyze-git.d.ts.map +1 -0
- package/dist/scripts/analyze-git.js +851 -0
- package/dist/scripts/analyze-git.js.map +1 -0
- package/dist/scripts/analyze-orders.d.ts +17 -0
- package/dist/scripts/analyze-orders.d.ts.map +1 -0
- package/dist/scripts/analyze-orders.js +1568 -0
- package/dist/scripts/analyze-orders.js.map +1 -0
- package/dist/scripts/diagnose-kibana-candles.d.ts +3 -0
- package/dist/scripts/diagnose-kibana-candles.d.ts.map +1 -0
- package/dist/scripts/diagnose-kibana-candles.js +82 -0
- package/dist/scripts/diagnose-kibana-candles.js.map +1 -0
- package/dist/scripts/diagnose-pool-history.d.ts +3 -0
- package/dist/scripts/diagnose-pool-history.d.ts.map +1 -0
- package/dist/scripts/diagnose-pool-history.js +296 -0
- package/dist/scripts/diagnose-pool-history.js.map +1 -0
- package/dist/scripts/divergence-calc.d.ts +26 -0
- package/dist/scripts/divergence-calc.d.ts.map +1 -0
- package/dist/scripts/divergence-calc.js +119 -0
- package/dist/scripts/divergence-calc.js.map +1 -0
- package/dist/scripts/generate_lp_chart.d.ts +31 -0
- package/dist/scripts/generate_lp_chart.d.ts.map +1 -0
- package/dist/scripts/generate_lp_chart.js +92 -0
- package/dist/scripts/generate_lp_chart.js.map +1 -0
- package/dist/scripts/generate_mainnet_corpus_report.d.ts +24 -0
- package/dist/scripts/generate_mainnet_corpus_report.d.ts.map +1 -0
- package/dist/scripts/generate_mainnet_corpus_report.js +237 -0
- package/dist/scripts/generate_mainnet_corpus_report.js.map +1 -0
- package/dist/scripts/generate_market_adapter_whitelist.d.ts +2 -0
- package/dist/scripts/generate_market_adapter_whitelist.d.ts.map +1 -0
- package/dist/scripts/generate_market_adapter_whitelist.js +111 -0
- package/dist/scripts/generate_market_adapter_whitelist.js.map +1 -0
- package/dist/scripts/migrate_bot_keys.d.ts +6 -0
- package/dist/scripts/migrate_bot_keys.d.ts.map +1 -0
- package/dist/scripts/migrate_bot_keys.js +301 -0
- package/dist/scripts/migrate_bot_keys.js.map +1 -0
- package/dist/scripts/native_release_gates.d.ts +3 -0
- package/dist/scripts/native_release_gates.d.ts.map +1 -0
- package/dist/scripts/native_release_gates.js +52 -0
- package/dist/scripts/native_release_gates.js.map +1 -0
- package/dist/scripts/print_grid.d.ts +16 -0
- package/dist/scripts/print_grid.d.ts.map +1 -0
- package/dist/scripts/print_grid.js +80 -0
- package/dist/scripts/print_grid.js.map +1 -0
- package/dist/scripts/run-tests.d.ts +2 -0
- package/dist/scripts/run-tests.d.ts.map +1 -0
- package/dist/scripts/run-tests.js +49 -0
- package/dist/scripts/run-tests.js.map +1 -0
- package/dist/scripts/sync-version.d.ts +14 -0
- package/dist/scripts/sync-version.d.ts.map +1 -0
- package/dist/scripts/sync-version.js +158 -0
- package/dist/scripts/sync-version.js.map +1 -0
- package/dist/scripts/test-credit-renewal.d.ts +2 -0
- package/dist/scripts/test-credit-renewal.d.ts.map +1 -0
- package/dist/scripts/test-credit-renewal.js +329 -0
- package/dist/scripts/test-credit-renewal.js.map +1 -0
- package/dist/scripts/update.d.ts +24 -0
- package/dist/scripts/update.d.ts.map +1 -0
- package/dist/scripts/update.js +650 -0
- package/dist/scripts/update.js.map +1 -0
- package/dist/scripts/validate_bots.d.ts +16 -0
- package/dist/scripts/validate_bots.d.ts.map +1 -0
- package/dist/scripts/validate_bots.js +106 -0
- package/dist/scripts/validate_bots.js.map +1 -0
- package/dist/scripts/verify-browser-bundle.d.ts +22 -0
- package/dist/scripts/verify-browser-bundle.d.ts.map +1 -0
- package/dist/scripts/verify-browser-bundle.js +339 -0
- package/dist/scripts/verify-browser-bundle.js.map +1 -0
- package/dist/unlock.d.ts +48 -0
- package/dist/unlock.d.ts.map +1 -0
- package/dist/unlock.js +925 -0
- package/dist/unlock.js.map +1 -0
- package/package.json +112 -0
- package/scripts/README.md +356 -0
- package/scripts/clear-all.sh +195 -0
- package/scripts/clear-logs.sh +95 -0
- package/scripts/clear-market-adapter.sh +196 -0
- package/scripts/clear-orders.sh +113 -0
- package/scripts/create-bot-symlinks.sh +51 -0
- package/scripts/postinstall.js +9 -0
- package/scripts/reset-settings.sh +118 -0
|
@@ -0,0 +1,4613 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* modules/dexbot_class.ts - DEXBot Core Engine
|
|
4
|
+
*
|
|
5
|
+
* Core trading bot implementation shared by bot.ts (single) and dexbot.ts (multi-bot).
|
|
6
|
+
* Implements complete grid trading bot lifecycle.
|
|
7
|
+
*
|
|
8
|
+
* Responsibilities:
|
|
9
|
+
* - Bot initialization and account setup
|
|
10
|
+
* - Order placement and batch operations
|
|
11
|
+
* - Fill processing and synchronization
|
|
12
|
+
* - Grid rebalancing and order rotation
|
|
13
|
+
* - Divergence detection and correction
|
|
14
|
+
* - State persistence and recovery
|
|
15
|
+
* - Market monitoring and health checks
|
|
16
|
+
*
|
|
17
|
+
* ===============================================================================
|
|
18
|
+
* CORE CLASS: DEXBot
|
|
19
|
+
* ===============================================================================
|
|
20
|
+
*
|
|
21
|
+
* LIFECYCLE METHODS:
|
|
22
|
+
* - constructor(config) - Initialize bot with configuration
|
|
23
|
+
* - run() - Start bot operation loop
|
|
24
|
+
* - shutdown() - Graceful shutdown
|
|
25
|
+
*
|
|
26
|
+
* FILL PROCESSING:
|
|
27
|
+
* - processFills() - Handle fill events
|
|
28
|
+
*
|
|
29
|
+
* SYNCHRONIZATION:
|
|
30
|
+
* - reconcileGrid() - Reconcile grid state with blockchain
|
|
31
|
+
*
|
|
32
|
+
* MONITORING:
|
|
33
|
+
* - monitorHealth() - Check bot health status
|
|
34
|
+
*
|
|
35
|
+
* ===============================================================================
|
|
36
|
+
*
|
|
37
|
+
* HELPER FUNCTIONS (module-level):
|
|
38
|
+
* - normalizeBotEntry() - Normalize bot configuration object
|
|
39
|
+
*
|
|
40
|
+
* ===============================================================================
|
|
41
|
+
*
|
|
42
|
+
* STATE MANAGEMENT:
|
|
43
|
+
* - Internal OrderManager maintains all state
|
|
44
|
+
* - Persists grid snapshots to profiles/orders/{botKey}.json
|
|
45
|
+
* - Recovers from persisted state on startup
|
|
46
|
+
* - Real-time synchronization with blockchain
|
|
47
|
+
*
|
|
48
|
+
* ERROR HANDLING:
|
|
49
|
+
* - Graceful error recovery
|
|
50
|
+
* - Automatic reconnection on connection loss
|
|
51
|
+
* - Anomaly detection and correction
|
|
52
|
+
* - Detailed logging for debugging
|
|
53
|
+
*
|
|
54
|
+
* ===============================================================================
|
|
55
|
+
*/
|
|
56
|
+
const { path } = require('./path_api');
|
|
57
|
+
const { BitShares, waitForConnected, onReconnect: registerReconnectHook } = require('./bitshares_client');
|
|
58
|
+
const { getStorage } = require('./storage');
|
|
59
|
+
const storage = getStorage();
|
|
60
|
+
const chainKeys = require('./chain_keys');
|
|
61
|
+
const { getKeyStore } = require('./key_store');
|
|
62
|
+
const chainOrders = require('./chain_orders');
|
|
63
|
+
const fundRegistry = require('./fund_registry');
|
|
64
|
+
const { BroadcastUncertainError } = require('./dexbot_credential_client');
|
|
65
|
+
const { OrderManager, grid: Grid } = require('./order');
|
|
66
|
+
const { retryPersistenceIfNeeded, initializeFeeCache, } = require('./order/utils/system');
|
|
67
|
+
const { hasExecutableActions, validateCreateTargetSlots } = require('./order/utils/validate');
|
|
68
|
+
const { buildCreateOrderArgs, buildCreateOpFingerprint, virtualizeOrder, correctAllPriceMismatches, convertToSpreadPlaceholder, buildOutsideInPairGroups, extractBatchOperationResults, buildFillKey, formatUnmatchedChainOrder } = require('./order/utils/order');
|
|
69
|
+
const { validateOrderSize } = require('./order/utils/math');
|
|
70
|
+
const { ProcessedFillStore, PROCESSED_FILL_PERSISTENCE_MODES } = require('./order/processed_fill_store');
|
|
71
|
+
const DexbotFillRuntime = require('./dexbot_fill_runtime');
|
|
72
|
+
const DexbotMaintenanceRuntime = require('./dexbot_maintenance_runtime');
|
|
73
|
+
const CreditRuntime = require('./credit_runtime');
|
|
74
|
+
const { ORDER_STATES, ORDER_TYPES, REBALANCE_STATES, COW_ACTIONS, TIMING, MAINTENANCE, FILL_PROCESSING, DAEMON_CODES, } = require('./constants');
|
|
75
|
+
const { PATHS, getRecalculateTriggerFile } = require('./paths');
|
|
76
|
+
const { attemptResumePersistedGridByPriceMatch, decideStartupGridAction, reconcileGridOrders } = require('./order/grid_reconcile');
|
|
77
|
+
const { AccountOrders } = require('./account_orders');
|
|
78
|
+
const { parseJsonWithComments } = require('./order/utils/system');
|
|
79
|
+
const { cloneWeightDistribution } = require('./order/utils/math');
|
|
80
|
+
const { normalizeBotEntry } = require('./bot_settings');
|
|
81
|
+
const Format = require('./order/format');
|
|
82
|
+
const { resolveBotRuntimeSettings } = require('./runtime_settings');
|
|
83
|
+
const PROFILES_BOTS_FILE = PATHS.PROFILES.BOTS_JSON;
|
|
84
|
+
const PROFILES_DIR = PATHS.PROFILES_DIR;
|
|
85
|
+
class DEXBot {
|
|
86
|
+
config;
|
|
87
|
+
_baseWeightDistribution;
|
|
88
|
+
account;
|
|
89
|
+
accountId = null;
|
|
90
|
+
privateKey;
|
|
91
|
+
manager;
|
|
92
|
+
accountOrders;
|
|
93
|
+
triggerFile;
|
|
94
|
+
_recentlyQueuedFills;
|
|
95
|
+
_fillCleanupCounter;
|
|
96
|
+
_fillDedupeWindowMs;
|
|
97
|
+
_fillRecordRetentionMs;
|
|
98
|
+
_processedFillPersistBatchMs;
|
|
99
|
+
_processedFillPersistBatchSize;
|
|
100
|
+
_processedFillStore;
|
|
101
|
+
_recentlyProcessedFills;
|
|
102
|
+
_pendingProcessedFillWrites;
|
|
103
|
+
_incomingFillQueue;
|
|
104
|
+
logPrefix;
|
|
105
|
+
_credentialDaemonWatchdogInterval;
|
|
106
|
+
_credentialDaemonDown;
|
|
107
|
+
_credentialRecoveryNeeded;
|
|
108
|
+
_credentialRecoveryInFlight;
|
|
109
|
+
_credentialDaemonWatchdogInFlight;
|
|
110
|
+
_staleCleanedOrderIds;
|
|
111
|
+
_staleCleanupRetentionMs;
|
|
112
|
+
_metrics;
|
|
113
|
+
_shuttingDown;
|
|
114
|
+
_shutdownStarted;
|
|
115
|
+
_shutdownPromise;
|
|
116
|
+
_blockchainFetchInterval;
|
|
117
|
+
_blockchainFetchInFlight;
|
|
118
|
+
_fillsUnsubscribe;
|
|
119
|
+
_triggerWatcher;
|
|
120
|
+
_triggerDebounceTimer;
|
|
121
|
+
_deferredGridResyncTimer;
|
|
122
|
+
_maintenanceIdleTimer;
|
|
123
|
+
_mainLoopActive;
|
|
124
|
+
_mainLoopPromise;
|
|
125
|
+
_creditRuntime;
|
|
126
|
+
_creditWatchdogInterval;
|
|
127
|
+
_batchInFlight;
|
|
128
|
+
_recoverySyncInFlight;
|
|
129
|
+
_lastTargetedDriftSyncAt;
|
|
130
|
+
_targetedDriftSyncCooldownMs;
|
|
131
|
+
_maintenanceCooldownCycles;
|
|
132
|
+
_lastGridActivityAt;
|
|
133
|
+
_currentCycleId;
|
|
134
|
+
_autoCancelOrphanCycleMarker;
|
|
135
|
+
_consecutiveConsumeFailures;
|
|
136
|
+
_consumeFailureFirstAt;
|
|
137
|
+
_reconnectUnregister;
|
|
138
|
+
_credentialRecoveryDeferredTimer;
|
|
139
|
+
_structuralGridResyncTimer;
|
|
140
|
+
_structuralGridResyncRunning;
|
|
141
|
+
_dustHealthCheckTimer;
|
|
142
|
+
_lastBroadcastHeartbeatAt;
|
|
143
|
+
_currentBatchId;
|
|
144
|
+
/**
|
|
145
|
+
* Create a new DEXBot instance
|
|
146
|
+
* @param {Object} config - Bot configuration from profiles/bots.json
|
|
147
|
+
* @param {Object} options - Optional settings
|
|
148
|
+
* @param {string} options.logPrefix - Prefix for console logs (e.g., "[bot.js]")
|
|
149
|
+
*/
|
|
150
|
+
constructor(config, options = {}) {
|
|
151
|
+
this._validateStartupConfig(config);
|
|
152
|
+
this.config = config;
|
|
153
|
+
this._baseWeightDistribution = cloneWeightDistribution(config.weightDistribution) || { sell: 0.5, buy: 0.5 };
|
|
154
|
+
this.account = null;
|
|
155
|
+
this.privateKey = null;
|
|
156
|
+
this.manager = null;
|
|
157
|
+
this.accountOrders = null;
|
|
158
|
+
this.triggerFile = getRecalculateTriggerFile(config.botKey);
|
|
159
|
+
this._recentlyQueuedFills = new Map();
|
|
160
|
+
this._fillCleanupCounter = 0;
|
|
161
|
+
const rs = resolveBotRuntimeSettings(this.config);
|
|
162
|
+
this.config.gridLimits = rs.gridLimits;
|
|
163
|
+
this.config.feeParams = rs.feeParams;
|
|
164
|
+
this.config.incrementBounds = rs.incrementBounds;
|
|
165
|
+
this.config.timing = rs.timing;
|
|
166
|
+
this.config.logging = rs.logging;
|
|
167
|
+
this._fillDedupeWindowMs = this.config.timing.FILL_DEDUPE_WINDOW_MS;
|
|
168
|
+
this._fillRecordRetentionMs = this.config.timing.FILL_RECORD_RETENTION_MS;
|
|
169
|
+
this._processedFillPersistBatchMs = TIMING.PROCESSED_FILL_PERSIST_BATCH_MS;
|
|
170
|
+
this._processedFillPersistBatchSize = TIMING.PROCESSED_FILL_PERSIST_BATCH_SIZE;
|
|
171
|
+
this._processedFillStore = new ProcessedFillStore({
|
|
172
|
+
batchMs: this._processedFillPersistBatchMs,
|
|
173
|
+
batchSize: this._processedFillPersistBatchSize,
|
|
174
|
+
warn: (message) => this._warn(message)
|
|
175
|
+
});
|
|
176
|
+
this._recentlyProcessedFills = this._processedFillStore.tracker;
|
|
177
|
+
this._pendingProcessedFillWrites = this._processedFillStore.pendingWrites;
|
|
178
|
+
this._incomingFillQueue = [];
|
|
179
|
+
this.logPrefix = options.logPrefix || '';
|
|
180
|
+
this._credentialDaemonWatchdogInterval = null;
|
|
181
|
+
this._credentialDaemonDown = false;
|
|
182
|
+
this._credentialRecoveryNeeded = false;
|
|
183
|
+
this._credentialRecoveryInFlight = false;
|
|
184
|
+
this._credentialDaemonWatchdogInFlight = false;
|
|
185
|
+
// TTL cache of order IDs freed by stale-order batch cleanup.
|
|
186
|
+
// If an orphan fill arrives for a stale-cleaned order within the retention
|
|
187
|
+
// window, we skip the credit to avoid double-counting freed capital.
|
|
188
|
+
this._staleCleanedOrderIds = new Map();
|
|
189
|
+
this._staleCleanupRetentionMs = Math.max(this._fillDedupeWindowMs || 0, 5 * 60 * 1000);
|
|
190
|
+
// Metrics for monitoring lock contention and fill processing
|
|
191
|
+
this._metrics = {
|
|
192
|
+
fillsProcessed: 0,
|
|
193
|
+
fillProcessingTimeMs: 0,
|
|
194
|
+
batchesExecuted: 0,
|
|
195
|
+
lockContentionEvents: 0,
|
|
196
|
+
maxQueueDepth: 0
|
|
197
|
+
};
|
|
198
|
+
// Shutdown state
|
|
199
|
+
this._shuttingDown = false;
|
|
200
|
+
this._shutdownStarted = false;
|
|
201
|
+
this._shutdownPromise = null;
|
|
202
|
+
// Runtime handles for graceful lifecycle management
|
|
203
|
+
this._blockchainFetchInterval = null;
|
|
204
|
+
this._blockchainFetchInFlight = false;
|
|
205
|
+
this._fillsUnsubscribe = null;
|
|
206
|
+
this._triggerWatcher = null;
|
|
207
|
+
this._triggerDebounceTimer = null;
|
|
208
|
+
this._deferredGridResyncTimer = null;
|
|
209
|
+
this._maintenanceIdleTimer = null;
|
|
210
|
+
this._mainLoopActive = false;
|
|
211
|
+
this._mainLoopPromise = null;
|
|
212
|
+
this._creditRuntime = null;
|
|
213
|
+
this._creditWatchdogInterval = null;
|
|
214
|
+
// Pipeline state flags (used by maintenance gating)
|
|
215
|
+
this._batchInFlight = false;
|
|
216
|
+
this._recoverySyncInFlight = false;
|
|
217
|
+
this._lastTargetedDriftSyncAt = 0;
|
|
218
|
+
this._targetedDriftSyncCooldownMs = this.config.timing.TARGETED_DRIFT_SYNC_COOLDOWN_MS;
|
|
219
|
+
this._maintenanceCooldownCycles = 0;
|
|
220
|
+
this._lastGridActivityAt = 0;
|
|
221
|
+
this._currentCycleId = 0;
|
|
222
|
+
this._autoCancelOrphanCycleMarker = null;
|
|
223
|
+
// Dust cancellation uses immediate on-chain cancel — no maps or timers needed.
|
|
224
|
+
this._dustHealthCheckTimer = null;
|
|
225
|
+
// Fill consumer watchdog: consecutive failure tracking.
|
|
226
|
+
// Reset on successful consumption; above _maxConsumeFailures, the
|
|
227
|
+
// re-schedule backs off and logs CRITICAL.
|
|
228
|
+
this._consecutiveConsumeFailures = 0;
|
|
229
|
+
this._consumeFailureFirstAt = 0;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Validate startup configuration to catch errors early.
|
|
233
|
+
* Ensures critical values are valid before bot starts.
|
|
234
|
+
* @param {Object} config - Configuration object to validate
|
|
235
|
+
* @throws {Error} If critical validation fails
|
|
236
|
+
* @private
|
|
237
|
+
*/
|
|
238
|
+
_validateStartupConfig(config) {
|
|
239
|
+
const errors = [];
|
|
240
|
+
// Validate startPrice is numeric or valid string mode
|
|
241
|
+
const startPrice = config.startPrice;
|
|
242
|
+
const validPriceModes = ['pool', 'book'];
|
|
243
|
+
const isPriceNumeric = typeof startPrice === 'number' && Number.isFinite(startPrice) && startPrice > 0;
|
|
244
|
+
const isPriceMode = typeof startPrice === 'string' && validPriceModes.includes(startPrice.toLowerCase());
|
|
245
|
+
if (!isPriceNumeric && !isPriceMode) {
|
|
246
|
+
errors.push(`startPrice must be a positive number or valid mode (${validPriceModes.join('/')}), got: ${startPrice}`);
|
|
247
|
+
}
|
|
248
|
+
// Validate assetA and assetB are present
|
|
249
|
+
if (!config.assetA || typeof config.assetA !== 'string') {
|
|
250
|
+
errors.push(`assetA must be a non-empty string, got: ${config.assetA}`);
|
|
251
|
+
}
|
|
252
|
+
if (!config.assetB || typeof config.assetB !== 'string') {
|
|
253
|
+
errors.push(`assetB must be a non-empty string, got: ${config.assetB}`);
|
|
254
|
+
}
|
|
255
|
+
// Validate incrementPercent
|
|
256
|
+
const increment = config.incrementPercent;
|
|
257
|
+
if (!Number.isFinite(increment) || increment <= 0 || increment > 100) {
|
|
258
|
+
errors.push(`incrementPercent must be between 0 and 100, got: ${increment}`);
|
|
259
|
+
}
|
|
260
|
+
// Throw all validation errors at once
|
|
261
|
+
if (errors.length > 0) {
|
|
262
|
+
throw new Error(`Config validation failed:\n${errors.map(e => ` - ${e}`).join('\n')}`);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Log a message to the console with the bot's prefix.
|
|
267
|
+
* @param {string} msg - The message to log.
|
|
268
|
+
* @param {string} [level='info'] - The log level ('debug', 'info', 'warn', 'error').
|
|
269
|
+
* @private
|
|
270
|
+
*/
|
|
271
|
+
_log(msg, level = 'info') {
|
|
272
|
+
if (level === 'warn') {
|
|
273
|
+
this._warn(msg);
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const line = this.logPrefix ? `${this.logPrefix} ${msg}` : msg;
|
|
277
|
+
const logger = this.manager?.logger;
|
|
278
|
+
if (logger && typeof logger.log === 'function') {
|
|
279
|
+
logger.log(line, level);
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
if (level === 'error') {
|
|
283
|
+
console.error(line);
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
console.log(line);
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Log a warning message to the console with the bot's prefix.
|
|
290
|
+
* @param {string} msg - The message to log.
|
|
291
|
+
* @private
|
|
292
|
+
*/
|
|
293
|
+
_warn(msg) {
|
|
294
|
+
const line = this.logPrefix ? `${this.logPrefix} ${msg}` : msg;
|
|
295
|
+
const logger = this.manager?.logger;
|
|
296
|
+
if (logger && typeof logger.log === 'function') {
|
|
297
|
+
logger.log(line, 'warn');
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
if (this.logPrefix) {
|
|
301
|
+
console.warn(line);
|
|
302
|
+
}
|
|
303
|
+
else {
|
|
304
|
+
console.warn(msg);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Persist the grid and trigger immediate recovery if validation fails.
|
|
309
|
+
* Used during startup to ensure bot begins in a stable state.
|
|
310
|
+
* @private
|
|
311
|
+
*/
|
|
312
|
+
async _persistAndRecoverIfNeeded() {
|
|
313
|
+
const validation = await this.manager.persistGrid();
|
|
314
|
+
if (!validation.isValid) {
|
|
315
|
+
this._warn(`Startup validation failed: ${validation.reason}. Triggering immediate recovery...`);
|
|
316
|
+
// Trigger centralized recovery (Hard Reset)
|
|
317
|
+
const recoveryValidation = await this.manager.accountant._performStateRecovery(this.manager);
|
|
318
|
+
if (recoveryValidation.isValid) {
|
|
319
|
+
this._log(`✓ Startup recovery successful. Persistent state restored.`);
|
|
320
|
+
await this.manager.persistGrid();
|
|
321
|
+
}
|
|
322
|
+
else {
|
|
323
|
+
this._warn(`Startup recovery failed: ${recoveryValidation.reason}. Bot proceeding with caution.`);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Get current pipeline signal state for congestion checks.
|
|
329
|
+
* @returns {{incomingFillQueueLength: number, shadowLocks: number, batchInFlight: boolean, recoveryInFlight: boolean, broadcasting: boolean}}
|
|
330
|
+
*/
|
|
331
|
+
_getPipelineSignals() {
|
|
332
|
+
this.manager?._cleanExpiredLocks?.();
|
|
333
|
+
return {
|
|
334
|
+
incomingFillQueueLength: this._incomingFillQueue.length,
|
|
335
|
+
shadowLocks: this.manager?.shadowOrderIds?.size || 0,
|
|
336
|
+
batchInFlight: this._batchInFlight,
|
|
337
|
+
recoveryInFlight: this._recoverySyncInFlight,
|
|
338
|
+
broadcasting: this.manager?._state?.isBroadcastingActive() || false
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Mark that grid activity occurred (updates idle timer).
|
|
343
|
+
* @param {string} [reason='activity'] - Reason for activity
|
|
344
|
+
* @returns {void}
|
|
345
|
+
*/
|
|
346
|
+
_markGridActivity(reason = 'activity') {
|
|
347
|
+
this._lastGridActivityAt = Date.now();
|
|
348
|
+
this.manager?.logger?.log?.(`[MAINT-IDLE] Activity observed: ${reason}`, 'debug');
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Trigger a full state recovery sync (fetch chain + sync from open orders + persist).
|
|
352
|
+
* @param {string} [reason='state recovery sync'] - Reason for recovery
|
|
353
|
+
* @returns {Promise<void>}
|
|
354
|
+
*/
|
|
355
|
+
async _triggerStateRecoverySync(reason = 'state recovery sync') {
|
|
356
|
+
if (!this.manager)
|
|
357
|
+
return;
|
|
358
|
+
if (this._recoverySyncInFlight) {
|
|
359
|
+
this.manager.logger.log(`[RECOVERY] Skipping duplicate recovery request: ${reason}`, 'warn');
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
this._recoverySyncInFlight = true;
|
|
363
|
+
try {
|
|
364
|
+
this.manager.logger.log(`Triggering state recovery sync (${reason})...`, 'info');
|
|
365
|
+
await this.manager.fetchAccountTotals(this.accountId);
|
|
366
|
+
const openOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
367
|
+
await this.manager.syncFromOpenOrders(openOrders, { skipAccounting: true, fillLockAlreadyHeld: true });
|
|
368
|
+
if (typeof this.manager.persistGrid === 'function') {
|
|
369
|
+
await this.manager.persistGrid();
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
finally {
|
|
373
|
+
this._recoverySyncInFlight = false;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Abort the current flow if an illegal state signal was raised.
|
|
378
|
+
* @param {string} flowContext - Description of the flow being aborted
|
|
379
|
+
* @returns {Promise<boolean>} True if flow was aborted
|
|
380
|
+
*/
|
|
381
|
+
async _abortFlowIfIllegalState(flowContext) {
|
|
382
|
+
const illegalSignal = this.manager?.consumeIllegalStateSignal?.();
|
|
383
|
+
if (!illegalSignal) {
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
this.manager.logger.log(`[HARD-ABORT] ${flowContext} aborted due to illegal state (${illegalSignal.context}): ${illegalSignal.message}`, 'error');
|
|
387
|
+
await this._triggerStateRecoverySync(`hard-abort ${flowContext}`);
|
|
388
|
+
this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
|
|
389
|
+
return true;
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Handle a hard abort from batch processing due to illegal state or accounting failure.
|
|
393
|
+
* @param {Error} err - The error that triggered the abort
|
|
394
|
+
* @param {string} [phase='batch processing'] - Phase description
|
|
395
|
+
* @param {number} [opsCount=0] - Number of operations in the batch
|
|
396
|
+
* @returns {Promise<Object>} Abort result object
|
|
397
|
+
*/
|
|
398
|
+
async _handleBatchHardAbort(err, phase = 'batch processing', opsCount = 0) {
|
|
399
|
+
const baseResult = { executed: false, hadRotation: false };
|
|
400
|
+
const opsInfo = opsCount > 0 ? ` with ${opsCount} ops` : '';
|
|
401
|
+
if (err?.code === 'ILLEGAL_ORDER_STATE') {
|
|
402
|
+
const illegalSignal = this.manager.consumeIllegalStateSignal?.();
|
|
403
|
+
await this._triggerStateRecoverySync(illegalSignal?.message || `illegal order state during ${phase}${opsInfo}`);
|
|
404
|
+
this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
|
|
405
|
+
return { ...baseResult, abortedForIllegalState: true };
|
|
406
|
+
}
|
|
407
|
+
if (err?.code === 'ACCOUNTING_COMMITMENT_FAILED') {
|
|
408
|
+
const accountingSignal = this.manager.consumeAccountingFailureSignal?.();
|
|
409
|
+
const reason = accountingSignal
|
|
410
|
+
? `accounting lock failure (${accountingSignal.side} ${Format.formatAmount8(accountingSignal.amount)}) during ${accountingSignal.context}`
|
|
411
|
+
: `accounting commitment lock failure during ${phase}${opsInfo}`;
|
|
412
|
+
await this._triggerStateRecoverySync(reason);
|
|
413
|
+
this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
|
|
414
|
+
return { ...baseResult, abortedForAccountingFailure: true };
|
|
415
|
+
}
|
|
416
|
+
return null;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Apply recoverable grid updates (order virtualisation) after a batch failure.
|
|
420
|
+
* @param {Array<Object>} updates - Array of order update objects
|
|
421
|
+
* @param {string} [context='recoverable-grid-update'] - Context label for logging
|
|
422
|
+
* @returns {Promise<number>} Number of updates applied
|
|
423
|
+
*/
|
|
424
|
+
async _applyRecoverableGridUpdates(updates, context = 'recoverable-grid-update') {
|
|
425
|
+
if (!this.manager || !Array.isArray(updates) || updates.length === 0) {
|
|
426
|
+
return 0;
|
|
427
|
+
}
|
|
428
|
+
let applied;
|
|
429
|
+
if (typeof this.manager.applyGridUpdateBatch === 'function') {
|
|
430
|
+
await this.manager.applyGridUpdateBatch(updates, context);
|
|
431
|
+
applied = updates.length;
|
|
432
|
+
}
|
|
433
|
+
else {
|
|
434
|
+
applied = 0;
|
|
435
|
+
for (const update of updates) {
|
|
436
|
+
if (typeof this.manager._updateOrder !== 'function')
|
|
437
|
+
break;
|
|
438
|
+
await this.manager._updateOrder(update, context);
|
|
439
|
+
applied++;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
// Persist master grid mutations applied outside COW (stale-order
|
|
443
|
+
// virtualization, size-drift corrections). These run in the COW catch
|
|
444
|
+
// handler where the success-path persistGrid is never reached.
|
|
445
|
+
if (applied > 0 && typeof this.manager.persistGrid === 'function') {
|
|
446
|
+
await this.manager.persistGrid();
|
|
447
|
+
}
|
|
448
|
+
return applied;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Recover from explicit stale order errors by virtualizing affected grid slots.
|
|
452
|
+
* @param {Set<string>|string[]} staleOrderIds - Set or array of stale chain order IDs
|
|
453
|
+
* @param {string} [reason='stale order cleanup'] - Reason for cleanup
|
|
454
|
+
* @returns {Promise<{executed: boolean, hadRotation: boolean, stale: boolean, recoveredByVirtualization?: boolean}>}
|
|
455
|
+
*/
|
|
456
|
+
async _recoverExplicitStaleOrders(staleOrderIds, reason = 'stale order cleanup') {
|
|
457
|
+
const staleIds = Array.from(staleOrderIds || []).filter(Boolean);
|
|
458
|
+
if (staleIds.length === 0) {
|
|
459
|
+
return { executed: false, hadRotation: false, stale: false };
|
|
460
|
+
}
|
|
461
|
+
this.manager.logger.log(`[COW] Stale order(s) detected: ${staleIds.join(', ')}. Applying targeted cleanup.`, 'warn');
|
|
462
|
+
const updates = [];
|
|
463
|
+
for (const [, gridOrder] of this.manager.orders.entries()) {
|
|
464
|
+
if (!gridOrder?.orderId || !staleOrderIds.has(gridOrder.orderId))
|
|
465
|
+
continue;
|
|
466
|
+
this._staleCleanedOrderIds.set(gridOrder.orderId, Date.now());
|
|
467
|
+
updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
|
|
468
|
+
}
|
|
469
|
+
// Register any stale IDs that had no matching grid slot
|
|
470
|
+
for (const orderId of staleIds) {
|
|
471
|
+
if (!this._staleCleanedOrderIds.has(orderId)) {
|
|
472
|
+
this._staleCleanedOrderIds.set(orderId, Date.now());
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if (updates.length > 0) {
|
|
476
|
+
await this._applyRecoverableGridUpdates(updates, reason);
|
|
477
|
+
}
|
|
478
|
+
else {
|
|
479
|
+
this.manager.logger.log(`[COW] No local grid slot matched stale order cleanup request (${staleIds.join(', ')}).`, 'debug');
|
|
480
|
+
}
|
|
481
|
+
return {
|
|
482
|
+
executed: false,
|
|
483
|
+
hadRotation: false,
|
|
484
|
+
stale: true,
|
|
485
|
+
recoveredByVirtualization: updates.length > 0
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
/**
|
|
489
|
+
* Recover from on-chain size drift detected during batch broadcast.
|
|
490
|
+
* @param {Error} err - The size drift error
|
|
491
|
+
* @returns {Promise<{executed: boolean, hadRotation: boolean, recoveredBySync: boolean, reason: string}>}
|
|
492
|
+
*/
|
|
493
|
+
async _recoverBatchSizeDrift(err, opContexts = []) {
|
|
494
|
+
// Try a targeted fix first: extract the affected order IDs from the
|
|
495
|
+
// operation contexts and correct them directly from chain. This
|
|
496
|
+
// avoids a full state recovery sync in the common single-order case.
|
|
497
|
+
const affectedOrderIds = this._extractSizeDriftOrderIds(opContexts);
|
|
498
|
+
if (affectedOrderIds.length > 0) {
|
|
499
|
+
this.manager.logger.log(`[COW] Targeted size-drift repair for ${affectedOrderIds.length} order(s): ${affectedOrderIds.join(', ')}`, 'debug');
|
|
500
|
+
const repaired = await this._targetedOrderRepair(affectedOrderIds);
|
|
501
|
+
if (repaired) {
|
|
502
|
+
return {
|
|
503
|
+
executed: false,
|
|
504
|
+
hadRotation: false,
|
|
505
|
+
recoveredBySync: true,
|
|
506
|
+
reason: 'ORDER_SIZE_DRIFT_TARGETED'
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
this.manager.logger.log('[COW] Targeted repair failed, falling back to full state recovery sync.', 'warn');
|
|
510
|
+
}
|
|
511
|
+
const reason = `recoverable size drift during COW batch: ${err.message}`;
|
|
512
|
+
this.manager.logger.log(`[COW] Recovering from on-chain size drift via recovery sync: ${err.message}`, 'warn');
|
|
513
|
+
await this._triggerStateRecoverySync(reason);
|
|
514
|
+
return {
|
|
515
|
+
executed: false,
|
|
516
|
+
hadRotation: false,
|
|
517
|
+
recoveredBySync: true,
|
|
518
|
+
reason: 'ORDER_SIZE_DRIFT'
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* Extract chain order IDs from opContexts for operations that could
|
|
523
|
+
* trigger a size-drift error (size-update and rotation update).
|
|
524
|
+
* @param {Array<Object>} opContexts
|
|
525
|
+
* @returns {string[]} Unique chain order IDs
|
|
526
|
+
*/
|
|
527
|
+
_extractSizeDriftOrderIds(opContexts) {
|
|
528
|
+
if (!Array.isArray(opContexts))
|
|
529
|
+
return [];
|
|
530
|
+
const ids = new Set();
|
|
531
|
+
for (const ctx of opContexts) {
|
|
532
|
+
if (ctx?.kind === 'size-update' && ctx?.updateInfo?.partialOrder?.orderId) {
|
|
533
|
+
ids.add(ctx.updateInfo.partialOrder.orderId);
|
|
534
|
+
}
|
|
535
|
+
else if (ctx?.kind === 'rotation' && ctx?.rotation?.oldOrder?.orderId) {
|
|
536
|
+
ids.add(ctx.rotation.oldOrder.orderId);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
return Array.from(ids);
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Attempt to repair size-drift for specific order IDs by reading their
|
|
543
|
+
* current on-chain state and correcting the local grid directly.
|
|
544
|
+
* Falls back gracefully (returns false) on any error.
|
|
545
|
+
* @param {string[]} orderIds
|
|
546
|
+
* @returns {Promise<boolean>} True if all affected orders were repaired
|
|
547
|
+
*/
|
|
548
|
+
async _targetedOrderRepair(orderIds) {
|
|
549
|
+
const { BitShares } = require('./bitshares_client');
|
|
550
|
+
const { virtualizeOrder } = require('./order/utils/order');
|
|
551
|
+
try {
|
|
552
|
+
const objects = await BitShares.db.get_objects(orderIds);
|
|
553
|
+
if (!Array.isArray(objects) || objects.length !== orderIds.length)
|
|
554
|
+
return false;
|
|
555
|
+
const updates = [];
|
|
556
|
+
for (let i = 0; i < orderIds.length; i++) {
|
|
557
|
+
const chainOrder = objects[i];
|
|
558
|
+
const gridOrder = Array.from(this.manager.orders.values())
|
|
559
|
+
.find((o) => o.orderId === orderIds[i]);
|
|
560
|
+
if (!gridOrder)
|
|
561
|
+
continue;
|
|
562
|
+
if (!chainOrder || typeof chainOrder.for_sale === 'undefined') {
|
|
563
|
+
// Order no longer exists on chain -> fully filled or cancelled.
|
|
564
|
+
updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
|
|
565
|
+
}
|
|
566
|
+
else {
|
|
567
|
+
const chainUnits = Number(chainOrder.for_sale);
|
|
568
|
+
if (Number.isFinite(chainUnits)) {
|
|
569
|
+
const { blockchainToFloat } = require('./order/utils/math');
|
|
570
|
+
const prec = gridOrder.type === ORDER_TYPES.SELL
|
|
571
|
+
? this.manager.assets.assetA.precision
|
|
572
|
+
: this.manager.assets.assetB.precision;
|
|
573
|
+
const floatSize = blockchainToFloat(chainUnits, prec);
|
|
574
|
+
if (floatSize !== gridOrder.size) {
|
|
575
|
+
updates.push({
|
|
576
|
+
id: gridOrder.id,
|
|
577
|
+
size: floatSize,
|
|
578
|
+
rawOnChain: chainOrder,
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
if (updates.length > 0) {
|
|
585
|
+
await this._applyRecoverableGridUpdates(updates, 'targeted-size-drift-repair');
|
|
586
|
+
}
|
|
587
|
+
return true;
|
|
588
|
+
}
|
|
589
|
+
catch (err) {
|
|
590
|
+
this.manager.logger.log(`[COW] Targeted order repair failed: ${err.message}`, 'debug');
|
|
591
|
+
return false;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* Initialize bot state from storage and blockchain.
|
|
596
|
+
* Consolidates common initialization logic for start() and startWithPrivateKey().
|
|
597
|
+
* @returns {{persistedGrid: Object, persistedBtsFeesOwed: number, persistedBoundaryIdx: number, persistedBtsBalance: number}}
|
|
598
|
+
* @private
|
|
599
|
+
*/
|
|
600
|
+
async _initializeStartupState() {
|
|
601
|
+
// Create AccountOrders with bot-specific file (one file per bot)
|
|
602
|
+
this.accountOrders = new AccountOrders({ botKey: this.config.botKey });
|
|
603
|
+
this._processedFillStore.configure({
|
|
604
|
+
accountOrders: this.accountOrders
|
|
605
|
+
});
|
|
606
|
+
// Load persisted processed fills to prevent reprocessing after restart
|
|
607
|
+
const loadedPersistedFills = this._processedFillStore.loadPersisted({
|
|
608
|
+
minTimestamp: Date.now() - this._fillRecordRetentionMs
|
|
609
|
+
});
|
|
610
|
+
if (loadedPersistedFills > 0) {
|
|
611
|
+
this._log(`Loaded ${loadedPersistedFills} persisted fill records to prevent reprocessing`);
|
|
612
|
+
}
|
|
613
|
+
// Ensure bot metadata is properly initialized in storage BEFORE any Grid operations
|
|
614
|
+
const raw = storage.readFile(PROFILES_BOTS_FILE);
|
|
615
|
+
const allBotsConfig = parseJsonWithComments(raw).bots || [];
|
|
616
|
+
const myBotConfig = allBotsConfig
|
|
617
|
+
.map((b, originalIdx) => b.active !== false ? normalizeBotEntry(b, originalIdx) : null)
|
|
618
|
+
.find(b => b && b.botKey === this.config.botKey);
|
|
619
|
+
if (myBotConfig) {
|
|
620
|
+
await this.accountOrders.syncMeta(myBotConfig);
|
|
621
|
+
}
|
|
622
|
+
if (!this.manager) {
|
|
623
|
+
const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
|
|
624
|
+
this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
|
|
625
|
+
this.manager.account = this.account;
|
|
626
|
+
this.manager.accountId = this.accountId;
|
|
627
|
+
this.manager.accountOrders = this.accountOrders;
|
|
628
|
+
}
|
|
629
|
+
this._wireStructuralGridResyncRequest();
|
|
630
|
+
this._wireProcessedFillTracking();
|
|
631
|
+
this.manager.startBootstrap();
|
|
632
|
+
try {
|
|
633
|
+
// Fetch account totals from blockchain at startup to initialize funds
|
|
634
|
+
try {
|
|
635
|
+
if (this.accountId && this.config.assetA && this.config.assetB) {
|
|
636
|
+
await this.manager._initializeAssets();
|
|
637
|
+
await this.manager.fetchAccountTotals(this.accountId);
|
|
638
|
+
this._log('Fetched blockchain account balances at startup');
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
catch (err) {
|
|
642
|
+
this._log(`Startup balance fetch FAILED: ${err.message}. Order sizing may be incorrect until next successful sync.`, 'error');
|
|
643
|
+
}
|
|
644
|
+
// Ensure fee cache is initialized before any fill processing that calls getAssetFees().
|
|
645
|
+
try {
|
|
646
|
+
await initializeFeeCache([this.config || {}], BitShares);
|
|
647
|
+
}
|
|
648
|
+
catch (err) {
|
|
649
|
+
this._log(`Fee cache initialization FAILED: ${err.message}. Fee calculations will use defaults until cache is refreshed.`, 'error');
|
|
650
|
+
}
|
|
651
|
+
const persistedGrid = this.accountOrders.loadGrid();
|
|
652
|
+
// CRITICAL REPAIR: Strip fake orderIds where orderId === id (e.g. "slot-0")
|
|
653
|
+
let repairedGrid = persistedGrid;
|
|
654
|
+
if (persistedGrid && persistedGrid.length > 0) {
|
|
655
|
+
let repairCount = 0;
|
|
656
|
+
repairedGrid = persistedGrid.map(order => {
|
|
657
|
+
if (order && order.orderId && order.orderId === order.id) {
|
|
658
|
+
repairCount++;
|
|
659
|
+
const repairedOrder = { ...order, orderId: '' };
|
|
660
|
+
if (repairedOrder.state === ORDER_STATES.ACTIVE || repairedOrder.state === ORDER_STATES.PARTIAL) {
|
|
661
|
+
repairedOrder.state = ORDER_STATES.VIRTUAL;
|
|
662
|
+
}
|
|
663
|
+
return repairedOrder;
|
|
664
|
+
}
|
|
665
|
+
return order;
|
|
666
|
+
});
|
|
667
|
+
if (repairCount > 0) {
|
|
668
|
+
this._log(`[REPAIR] Stripped ${repairCount} fake orderId(s) from persisted grid to restore rebalancing logic.`);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
const persistedBtsFeesOwed = this.accountOrders.loadBtsFeesOwed();
|
|
672
|
+
const persistedBoundaryIdx = this.accountOrders.loadBoundaryIdx();
|
|
673
|
+
const persistedBtsBalance = this.accountOrders.loadBtsBalance();
|
|
674
|
+
return {
|
|
675
|
+
persistedGrid: repairedGrid,
|
|
676
|
+
persistedBtsFeesOwed,
|
|
677
|
+
persistedBoundaryIdx,
|
|
678
|
+
persistedBtsBalance,
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
finally {
|
|
682
|
+
this.manager.finishBootstrap();
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
/**
|
|
686
|
+
* Wire processed fill tracking into the manager.
|
|
687
|
+
* @returns {void}
|
|
688
|
+
*/
|
|
689
|
+
_wireProcessedFillTracking() {
|
|
690
|
+
return DexbotFillRuntime.wireProcessedFillTracking(this);
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Flush pending processed fill persistence to disk.
|
|
694
|
+
* @param {string} [reason='manual'] - Reason for flushing
|
|
695
|
+
* @param {Object} [options={}] - Flush options
|
|
696
|
+
* @returns {Promise<void>}
|
|
697
|
+
*/
|
|
698
|
+
async _flushProcessedFillPersistence(reason = 'manual', options = {}) {
|
|
699
|
+
return DexbotFillRuntime.flushProcessedFillPersistence(this, reason, options);
|
|
700
|
+
}
|
|
701
|
+
/**
|
|
702
|
+
* Flush persistence for specific fill keys.
|
|
703
|
+
* @param {Set<string>|string[]} fillKeys - Fill keys to persist
|
|
704
|
+
* @param {string} [reason='manual-selected'] - Reason for flushing
|
|
705
|
+
* @param {Object} [options={}] - Flush options
|
|
706
|
+
* @returns {Promise<void>}
|
|
707
|
+
*/
|
|
708
|
+
async _flushProcessedFillPersistenceForKeys(fillKeys, reason = 'manual-selected', options = {}) {
|
|
709
|
+
return DexbotFillRuntime.flushProcessedFillPersistenceForKeys(this, fillKeys, reason, options);
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* Discard pending persistence for specific fill keys.
|
|
713
|
+
* @param {string[]|Set<string>} fillKeys - Fill keys to discard
|
|
714
|
+
* @returns {void}
|
|
715
|
+
*/
|
|
716
|
+
_discardPendingProcessedFillPersistence(fillKeys) {
|
|
717
|
+
return DexbotFillRuntime.discardPendingProcessedFillPersistence(this, fillKeys);
|
|
718
|
+
}
|
|
719
|
+
/**
|
|
720
|
+
* Build a fallback deduplication key for an orphan fill (when standard keys are unavailable).
|
|
721
|
+
* @param {Object} fill - Fill event object
|
|
722
|
+
* @returns {string|null} Fallback key or null
|
|
723
|
+
*/
|
|
724
|
+
_buildOrphanFillFallbackKey(fill) {
|
|
725
|
+
return DexbotFillRuntime.buildOrphanFillFallbackKey(this, fill);
|
|
726
|
+
}
|
|
727
|
+
_isNewFillKey(fillKey, processedFillKeys, label = '') {
|
|
728
|
+
const now = Date.now();
|
|
729
|
+
if (this._recentlyQueuedFills.has(fillKey)) {
|
|
730
|
+
const lastProcessed = this._recentlyQueuedFills.get(fillKey);
|
|
731
|
+
if (now - lastProcessed < this._fillDedupeWindowMs) {
|
|
732
|
+
if (label) {
|
|
733
|
+
this.manager?.logger?.log?.(`${label} Skipping duplicate fill (processed ${now - lastProcessed}ms ago)`, 'debug');
|
|
734
|
+
}
|
|
735
|
+
return false;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
if (processedFillKeys.has(fillKey))
|
|
739
|
+
return false;
|
|
740
|
+
processedFillKeys.add(fillKey);
|
|
741
|
+
this._recentlyQueuedFills.set(fillKey, now);
|
|
742
|
+
return true;
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* Apply replay-safe fill accounting using a provided fill key.
|
|
746
|
+
* @param {Object} fill - Fill event object
|
|
747
|
+
* @param {import('./types').FillOperationData} fillOp - Fill operation data
|
|
748
|
+
* @param {Object} [options={}] - Options
|
|
749
|
+
* @param {string} [options.missingKeyMessage]
|
|
750
|
+
* @param {string} [options.fallbackKeyMessage]
|
|
751
|
+
* @param {string} [options.replayMessage]
|
|
752
|
+
* @param {string} [options.errorMessage]
|
|
753
|
+
* @param {Object} [options.logger]
|
|
754
|
+
* @param {string} [options.missingKeyLevel='warn']
|
|
755
|
+
* @param {string} [options.fallbackKeyLevel='warn']
|
|
756
|
+
* @param {string} [options.replayLevel='debug']
|
|
757
|
+
* @param {string} [options.persistenceMode='immediate']
|
|
758
|
+
* @param {boolean} [options.allowOrphanFallbackKey=false]
|
|
759
|
+
* @returns {Promise<import('./types').ReplaySafeFillResult>}
|
|
760
|
+
*/
|
|
761
|
+
async _applyReplaySafeFillAccounting(fill, fillOp, { missingKeyMessage, fallbackKeyMessage, replayMessage, errorMessage, logger = this.manager?.logger, missingKeyLevel = 'warn', fallbackKeyLevel = 'warn', replayLevel = 'debug', persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.IMMEDIATE, allowOrphanFallbackKey = false } = {}) {
|
|
762
|
+
return DexbotFillRuntime.applyReplaySafeFillAccounting(this, fill, fillOp, {
|
|
763
|
+
missingKeyMessage,
|
|
764
|
+
fallbackKeyMessage,
|
|
765
|
+
replayMessage,
|
|
766
|
+
errorMessage,
|
|
767
|
+
logger,
|
|
768
|
+
missingKeyLevel,
|
|
769
|
+
fallbackKeyLevel,
|
|
770
|
+
replayLevel,
|
|
771
|
+
persistenceMode,
|
|
772
|
+
allowOrphanFallbackKey
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
/**
|
|
776
|
+
* Apply replay-safe fill accounting for tracked fills (those with a valid grid order).
|
|
777
|
+
* @param {Object} fill - Fill event object
|
|
778
|
+
* @param {import('./types').FillOperationData} fillOp - Fill operation data
|
|
779
|
+
* @param {Object} [options={}]
|
|
780
|
+
* @param {string} [options.context]
|
|
781
|
+
* @param {Object} [options.logger]
|
|
782
|
+
* @param {string} [options.replayMessage]
|
|
783
|
+
* @param {string} [options.persistenceMode='batched']
|
|
784
|
+
* @returns {Promise<import('./types').ReplaySafeFillResult>}
|
|
785
|
+
*/
|
|
786
|
+
async _applyReplaySafeTrackedFillAccounting(fill, fillOp, { context, logger = this.manager?.logger, replayMessage, persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.BATCHED } = {}) {
|
|
787
|
+
return DexbotFillRuntime.applyReplaySafeTrackedFillAccounting(this, fill, fillOp, {
|
|
788
|
+
context,
|
|
789
|
+
logger,
|
|
790
|
+
replayMessage,
|
|
791
|
+
persistenceMode
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
/**
|
|
795
|
+
* Apply replay-safe fill accounting for orphan fills (grid order not found).
|
|
796
|
+
* @param {Object} fill - Fill event object
|
|
797
|
+
* @param {import('./types').FillOperationData} fillOp - Fill operation data
|
|
798
|
+
* @param {Object} [options={}]
|
|
799
|
+
* @param {string} [options.context]
|
|
800
|
+
* @param {Object} [options.logger]
|
|
801
|
+
* @param {string} [options.replayMessage]
|
|
802
|
+
* @param {string} [options.persistenceMode='immediate']
|
|
803
|
+
* @returns {Promise<import('./types').ReplaySafeFillResult>}
|
|
804
|
+
*/
|
|
805
|
+
async _applyReplaySafeOrphanFillAccounting(fill, fillOp, { context, logger = this.manager?.logger, replayMessage, persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.IMMEDIATE } = {}) {
|
|
806
|
+
return DexbotFillRuntime.applyReplaySafeOrphanFillAccounting(this, fill, fillOp, {
|
|
807
|
+
context,
|
|
808
|
+
logger,
|
|
809
|
+
replayMessage,
|
|
810
|
+
persistenceMode
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
/**
|
|
814
|
+
* Refresh dynamic weight distribution from market adapter.
|
|
815
|
+
* @param {string} [context='runtime'] - Context label for logging
|
|
816
|
+
* @returns {import('./types').DynamicWeightRefreshResult|null}
|
|
817
|
+
*/
|
|
818
|
+
_refreshDynamicWeightDistribution(context = 'runtime') {
|
|
819
|
+
return DexbotMaintenanceRuntime.refreshDynamicWeightDistribution(this, context);
|
|
820
|
+
}
|
|
821
|
+
/**
|
|
822
|
+
* Finalize the bot startup after account and initial grid sync are complete.
|
|
823
|
+
* Consolidates common logic for start() and startWithPrivateKey().
|
|
824
|
+
* @param {Object} startupState - The startup state from _initializeStartupState.
|
|
825
|
+
* @private
|
|
826
|
+
*/
|
|
827
|
+
async _finishStartupSequence(startupState) {
|
|
828
|
+
let { persistedGrid, persistedBtsFeesOwed, persistedBoundaryIdx, persistedBtsBalance, } = startupState;
|
|
829
|
+
try {
|
|
830
|
+
// CRITICAL: Activate fill listener EARLY - before ANY operations that place orders
|
|
831
|
+
// This ensures fills during trigger reset and grid initialization are captured
|
|
832
|
+
if (typeof this._fillsUnsubscribe === 'function') {
|
|
833
|
+
await this._fillsUnsubscribe().catch(() => { });
|
|
834
|
+
}
|
|
835
|
+
this._fillsUnsubscribe = await chainOrders.listenForFills(this.account || undefined, this._createFillCallback(chainOrders));
|
|
836
|
+
if (typeof this._fillsUnsubscribe !== 'function') {
|
|
837
|
+
this._warn('Fill listener did not provide an unsubscribe handler. Shutdown cleanup may be incomplete.');
|
|
838
|
+
this._fillsUnsubscribe = null;
|
|
839
|
+
}
|
|
840
|
+
this._log('Fill listener activated (ready to process fills during startup)');
|
|
841
|
+
// Register reconnection callback for safety-net sync after websocket reconnect
|
|
842
|
+
if (!this._reconnectUnregister) {
|
|
843
|
+
this._reconnectUnregister = registerReconnectHook(() => {
|
|
844
|
+
this._log('Blockchain connection re-established; scheduling safety-net sync');
|
|
845
|
+
const runSafetyNetSync = async () => {
|
|
846
|
+
if (this.manager && this.accountId && !this._shuttingDown && !this.config.dryRun) {
|
|
847
|
+
// Cap the entire safety-net sync at TIMING.SAFETY_NET_SYNC_TIMEOUT_MS so it can
|
|
848
|
+
// never hold _fillProcessingLock longer than the
|
|
849
|
+
// 20s shutdown lock timeout. readOpenOrders +
|
|
850
|
+
// synchronizeWithChain + batch + persist can be
|
|
851
|
+
// ~45s worst case without a cap, which would
|
|
852
|
+
// stall shutdown until the 20s timeout fires.
|
|
853
|
+
const safetyNetTimeoutMs = this.config.timing?.SAFETY_NET_SYNC_TIMEOUT_MS;
|
|
854
|
+
let safetyNetTimer;
|
|
855
|
+
const workPromise = this.manager._fillProcessingLock.acquire(async () => {
|
|
856
|
+
if (this._shuttingDown)
|
|
857
|
+
return;
|
|
858
|
+
const chainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
859
|
+
if (this._shuttingDown)
|
|
860
|
+
return;
|
|
861
|
+
const syncResult = await this.manager.synchronizeWithChain(chainOpenOrders, 'readOpenOrders', { fillLockAlreadyHeld: true });
|
|
862
|
+
if (this._shuttingDown)
|
|
863
|
+
return;
|
|
864
|
+
if (syncResult?.filledOrders?.length > 0) {
|
|
865
|
+
this._log(`Post-reconnect sync: ${syncResult.filledOrders.length} grid order(s) found filled.`, 'info');
|
|
866
|
+
await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'post-reconnect sync fill');
|
|
867
|
+
if (this._shuttingDown)
|
|
868
|
+
return;
|
|
869
|
+
}
|
|
870
|
+
await this.manager.persistGrid();
|
|
871
|
+
// Cancel any dust created by reconnect fills immediately.
|
|
872
|
+
if (!this._shuttingDown) {
|
|
873
|
+
try {
|
|
874
|
+
const reconnectHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
875
|
+
await this._cancelDustOrders({
|
|
876
|
+
buy: reconnectHealth.buyDustOrders,
|
|
877
|
+
sell: reconnectHealth.sellDustOrders,
|
|
878
|
+
fillLockAlreadyHeld: true,
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
catch (_dustErr) {
|
|
882
|
+
this._warn(`[RECONNECT] Dust cancel failed: ${_dustErr.message}`);
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
});
|
|
886
|
+
try {
|
|
887
|
+
await Promise.race([
|
|
888
|
+
workPromise,
|
|
889
|
+
new Promise((_, reject) => {
|
|
890
|
+
safetyNetTimer = setTimeout(() => reject(new Error(`Safety-net sync exceeded ${safetyNetTimeoutMs}ms cap`)), safetyNetTimeoutMs);
|
|
891
|
+
})
|
|
892
|
+
]);
|
|
893
|
+
}
|
|
894
|
+
catch (capErr) {
|
|
895
|
+
const fallback = await Promise.race([
|
|
896
|
+
workPromise.then(() => ({ ok: true })),
|
|
897
|
+
new Promise(resolve => setTimeout(() => resolve({ ok: false }), 0))
|
|
898
|
+
]);
|
|
899
|
+
if (fallback.ok) {
|
|
900
|
+
this._log(`Safety-net sync completed despite timeout — ignoring spurious error.`, 'info');
|
|
901
|
+
}
|
|
902
|
+
else {
|
|
903
|
+
this._warn(`Post-reconnect safety-net sync aborted: ${capErr?.message || capErr}`);
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
finally {
|
|
907
|
+
if (safetyNetTimer)
|
|
908
|
+
clearTimeout(safetyNetTimer);
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
};
|
|
912
|
+
// The setImmediate callback returns a Promise; we MUST attach
|
|
913
|
+
// a .catch so a rejection here does not propagate to
|
|
914
|
+
// process.on('unhandledRejection') and tear down the bot.
|
|
915
|
+
setImmediate(() => {
|
|
916
|
+
runSafetyNetSync().catch(err => {
|
|
917
|
+
try {
|
|
918
|
+
this._warn('Post-reconnect safety-net sync failed: ' + (err?.message || err));
|
|
919
|
+
}
|
|
920
|
+
catch (_warnErr) {
|
|
921
|
+
// _warn itself inaccessible — last line of defense.
|
|
922
|
+
}
|
|
923
|
+
});
|
|
924
|
+
});
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
// CRITICAL: Handle any pending trigger file reset FIRST before any other startup operations
|
|
928
|
+
const hadTriggerReset = await this._handlePendingTriggerReset();
|
|
929
|
+
// CRITICAL: After trigger reset, skip normal startup - grid is already fully initialized
|
|
930
|
+
// The trigger reset already did: grid init, order placement, sync, and persistence
|
|
931
|
+
if (hadTriggerReset) {
|
|
932
|
+
this._log('Trigger reset completed. Skipping normal startup grid initialization.');
|
|
933
|
+
// Post-bootstrap validation and fill processing
|
|
934
|
+
await this.manager._fillProcessingLock.acquire(async () => {
|
|
935
|
+
// STEP 1: Check for fills that occurred during trigger reset
|
|
936
|
+
// These are orders that got filled while Grid.recalculateGrid() was running.
|
|
937
|
+
// The filled slots need new orders placed on them.
|
|
938
|
+
if (this._incomingFillQueue.length > 0) {
|
|
939
|
+
this._log(`[POST-RESET] ${this._incomingFillQueue.length} fill(s) detected during trigger reset. Processing...`);
|
|
940
|
+
// Process fills - this will place new orders on the filled slots
|
|
941
|
+
// Use normal fill processing since bootstrap is complete
|
|
942
|
+
const fills = this._incomingFillQueue.splice(0);
|
|
943
|
+
const processedFillKeys = new Set();
|
|
944
|
+
let requiresOpenOrdersSync = false;
|
|
945
|
+
for (const fill of fills) {
|
|
946
|
+
if (!fill || fill.op?.[0] !== 4)
|
|
947
|
+
continue;
|
|
948
|
+
const fillOp = fill.op[1];
|
|
949
|
+
const gridOrder = this.manager.orders.get(fillOp.order_id) ||
|
|
950
|
+
Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
|
|
951
|
+
if (!gridOrder) {
|
|
952
|
+
// CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
|
|
953
|
+
// This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
|
|
954
|
+
let orphanFillKey = buildFillKey(fill);
|
|
955
|
+
if (!orphanFillKey) {
|
|
956
|
+
orphanFillKey = this._buildOrphanFillFallbackKey(fill);
|
|
957
|
+
}
|
|
958
|
+
if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[POST-RESET]')) {
|
|
959
|
+
continue;
|
|
960
|
+
}
|
|
961
|
+
this._log(`[POST-RESET] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
|
|
962
|
+
const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
|
|
963
|
+
context: 'POST-RESET',
|
|
964
|
+
logger: { log: this._log.bind(this) },
|
|
965
|
+
replayMessage: (op) => `[POST-RESET] Replay detected for orphan fill ${op.order_id}; skipping duplicate credit`
|
|
966
|
+
});
|
|
967
|
+
if (accountingResult.status === 'missing_key') {
|
|
968
|
+
requiresOpenOrdersSync = true;
|
|
969
|
+
}
|
|
970
|
+
continue;
|
|
971
|
+
}
|
|
972
|
+
this._log(`[POST-RESET] Processing fill for ${gridOrder.type} order ${gridOrder.id} at price ${gridOrder.price}`);
|
|
973
|
+
const trackedFillKey = buildFillKey(fill);
|
|
974
|
+
if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[POST-RESET]')) {
|
|
975
|
+
continue;
|
|
976
|
+
}
|
|
977
|
+
const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
|
|
978
|
+
context: 'POST-RESET',
|
|
979
|
+
logger: { log: this._log.bind(this) },
|
|
980
|
+
replayMessage: (op) => `[POST-RESET] Replay detected for ${op.order_id}; skipping duplicate rebalance`
|
|
981
|
+
});
|
|
982
|
+
if (accountingResult.status === 'missing_key') {
|
|
983
|
+
requiresOpenOrdersSync = true;
|
|
984
|
+
continue;
|
|
985
|
+
}
|
|
986
|
+
if (accountingResult.status !== 'applied') {
|
|
987
|
+
continue;
|
|
988
|
+
}
|
|
989
|
+
// Process this fill through the full rebalance pipeline
|
|
990
|
+
// This will shift the boundary and place a new order on the filled slot
|
|
991
|
+
const result = await this._processFillsWithBatching([gridOrder], new Set(), `[POST-RESET] fill ${gridOrder.id}`);
|
|
992
|
+
if (result.aborted) {
|
|
993
|
+
this._warn('[POST-RESET] Aborted batch due to illegal state; skipping grid persistence this cycle');
|
|
994
|
+
continue;
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
if (requiresOpenOrdersSync) {
|
|
998
|
+
this._log('[POST-RESET] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
|
|
999
|
+
const postResetChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1000
|
+
const syncResult = await this.manager.syncFromOpenOrders(postResetChainOpenOrders, { fillLockAlreadyHeld: true });
|
|
1001
|
+
if (syncResult.filledOrders?.length > 0) {
|
|
1002
|
+
await this._processFillsWithBatching(syncResult.filledOrders, new Set(), '[POST-RESET] open-orders fallback');
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
await this._flushProcessedFillPersistence('post-reset-batch');
|
|
1006
|
+
await this.manager.persistGrid();
|
|
1007
|
+
}
|
|
1008
|
+
// STEP 2: Refresh chain truth before spread correction. Trigger
|
|
1009
|
+
// reset can create/cancel orders and fills can arrive while the
|
|
1010
|
+
// reset is running; spread decisions must not use stale local grid.
|
|
1011
|
+
const { aborted: postResetAborted, hasUnmatched: postResetUnmatched } = await this._syncOpenOrdersAndProcessFills('[POST-RESET] pre-spread');
|
|
1012
|
+
if (postResetUnmatched) {
|
|
1013
|
+
this._warn(`[POST-RESET] Skipping spread correction: ${postResetUnmatched} unmatched chain order(s) require maintenance reconciliation`);
|
|
1014
|
+
}
|
|
1015
|
+
// STEP 3: Spread check AFTER fills are processed and chain truth refreshed
|
|
1016
|
+
await this.manager.recalculateFunds();
|
|
1017
|
+
if (!postResetAborted && !postResetUnmatched) {
|
|
1018
|
+
const spreadResult = await this.manager.checkSpreadCondition(BitShares, this.updateOrdersOnChainPlan.bind(this));
|
|
1019
|
+
if (spreadResult && spreadResult.ordersPlaced > 0) {
|
|
1020
|
+
this._log(`✓ Spread correction after trigger reset: ${spreadResult.ordersPlaced} order(s) placed`);
|
|
1021
|
+
await this._persistAndRecoverIfNeeded();
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
// Cancel any dust created by post-reset fills immediately.
|
|
1025
|
+
if (!this._shuttingDown) {
|
|
1026
|
+
try {
|
|
1027
|
+
const postResetHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
1028
|
+
await this._cancelDustOrders({
|
|
1029
|
+
buy: postResetHealth.buyDustOrders,
|
|
1030
|
+
sell: postResetHealth.sellDustOrders,
|
|
1031
|
+
fillLockAlreadyHeld: true,
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
catch (_dustErr) {
|
|
1035
|
+
this._warn(`[POST-RESET] Dust cancel failed: ${_dustErr.message}`);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
this._log('Bootstrap phase complete - fill processing resumed', 'info');
|
|
1039
|
+
});
|
|
1040
|
+
await this._setupTriggerFileDetection();
|
|
1041
|
+
await this._setupCreditRuntime();
|
|
1042
|
+
await this._refreshAndSyncCreditRuntime();
|
|
1043
|
+
this._setupBlockchainFetchInterval();
|
|
1044
|
+
this._setupCreditWatchdogInterval();
|
|
1045
|
+
this._setupCredentialDaemonWatchdogInterval();
|
|
1046
|
+
this._setupDustHealthCheckInterval();
|
|
1047
|
+
if (this._isOpenOrdersSyncLoopEnabled()) {
|
|
1048
|
+
this._startOpenOrdersSyncLoop();
|
|
1049
|
+
}
|
|
1050
|
+
else {
|
|
1051
|
+
this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
|
|
1052
|
+
}
|
|
1053
|
+
this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
|
|
1054
|
+
return; // Skip normal startup path
|
|
1055
|
+
}
|
|
1056
|
+
// Restore persisted BTS fee
|
|
1057
|
+
// SAFE: Done at startup before orders are created, and within fill lock when needed
|
|
1058
|
+
this.manager.resetFunds();
|
|
1059
|
+
// CRITICAL FIX: Restore BTS fees owed from persistence
|
|
1060
|
+
if (persistedBtsFeesOwed && persistedBtsFeesOwed > 0) {
|
|
1061
|
+
this.manager.funds.btsFeesOwed = Number(persistedBtsFeesOwed);
|
|
1062
|
+
}
|
|
1063
|
+
// Restore BTS balance for non-BTS pairs
|
|
1064
|
+
if (this.config.assetA !== 'BTS' && this.config.assetB !== 'BTS') {
|
|
1065
|
+
if (persistedBtsBalance && typeof persistedBtsBalance === 'object') {
|
|
1066
|
+
this.manager.btsBalance = {
|
|
1067
|
+
free: persistedBtsBalance.free || 0,
|
|
1068
|
+
total: persistedBtsBalance.total || 0,
|
|
1069
|
+
locked: persistedBtsBalance.locked || 0
|
|
1070
|
+
};
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
if (!this.config.dryRun && !this.accountId) {
|
|
1074
|
+
throw new Error('Cannot start bot without a resolved account ID');
|
|
1075
|
+
}
|
|
1076
|
+
// Use this.accountId which was set during initialize()
|
|
1077
|
+
const chainOpenOrders = this.config.dryRun ? [] : await chainOrders.readOpenOrders(this.accountId);
|
|
1078
|
+
let shouldRegenerate = false;
|
|
1079
|
+
if (!persistedGrid || persistedGrid.length === 0) {
|
|
1080
|
+
shouldRegenerate = true;
|
|
1081
|
+
this._log('No persisted grid found. Generating new grid.');
|
|
1082
|
+
}
|
|
1083
|
+
else {
|
|
1084
|
+
await this.manager._initializeAssets();
|
|
1085
|
+
const decision = await decideStartupGridAction({
|
|
1086
|
+
persistedGrid,
|
|
1087
|
+
chainOpenOrders,
|
|
1088
|
+
manager: this.manager,
|
|
1089
|
+
logger: { log: (msg) => this._log(msg) },
|
|
1090
|
+
storeGrid: async (orders) => {
|
|
1091
|
+
// Pass the snapshot directly to persistGrid so the live
|
|
1092
|
+
// `manager.orders` map is not briefly swapped (which would
|
|
1093
|
+
// leave the _ordersByState/_ordersByType indexes inconsistent
|
|
1094
|
+
// with the map for the duration of the persist call).
|
|
1095
|
+
await this.manager.persistGrid(orders);
|
|
1096
|
+
},
|
|
1097
|
+
attemptResumeFn: attemptResumePersistedGridByPriceMatch,
|
|
1098
|
+
});
|
|
1099
|
+
shouldRegenerate = decision.shouldRegenerate;
|
|
1100
|
+
if (shouldRegenerate && chainOpenOrders.length === 0) {
|
|
1101
|
+
this._log('Persisted grid found, but no matching active orders on-chain. Generating new grid.');
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
// Restore BTS fees owed ONLY if we're NOT regenerating the grid
|
|
1105
|
+
if (!shouldRegenerate) {
|
|
1106
|
+
// CRITICAL: Restore BTS fees owed from blockchain operations
|
|
1107
|
+
if (persistedBtsFeesOwed > 0) {
|
|
1108
|
+
this.manager.funds.btsFeesOwed = persistedBtsFeesOwed;
|
|
1109
|
+
this._log(`✓ Restored BTS fees owed: ${Format.formatAmount8(persistedBtsFeesOwed)} BTS`);
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
else {
|
|
1113
|
+
this._log(`ℹ Grid regenerating - resetting BTS fees to clean state`);
|
|
1114
|
+
this.manager.funds.btsFeesOwed = 0;
|
|
1115
|
+
}
|
|
1116
|
+
// CRITICAL: Use fill lock during ENTIRE startup synchronization to prevent races.
|
|
1117
|
+
// This includes grid init, finishBootstrap, and maintenance - all in one atomic block.
|
|
1118
|
+
// Lock order: _fillProcessingLock → _divergenceLock (canonical order, same as _consumeFillQueue)
|
|
1119
|
+
await this.manager._fillProcessingLock.acquire(async () => {
|
|
1120
|
+
try {
|
|
1121
|
+
this._refreshDynamicWeightDistribution('startup');
|
|
1122
|
+
if (shouldRegenerate) {
|
|
1123
|
+
await this.manager._initializeAssets();
|
|
1124
|
+
if (Array.isArray(chainOpenOrders) && chainOpenOrders.length > 0) {
|
|
1125
|
+
this._log('Generating new grid and syncing with existing on-chain orders...');
|
|
1126
|
+
await Grid.initializeGrid(this.manager);
|
|
1127
|
+
await this.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true, fillLockAlreadyHeld: true });
|
|
1128
|
+
const rebalanceResult = await reconcileGridOrders({
|
|
1129
|
+
manager: this.manager,
|
|
1130
|
+
config: this.config,
|
|
1131
|
+
account: this.account,
|
|
1132
|
+
privateKey: this.privateKey,
|
|
1133
|
+
chainOrders,
|
|
1134
|
+
chainOpenOrders,
|
|
1135
|
+
fillLockAlreadyHeld: true,
|
|
1136
|
+
});
|
|
1137
|
+
await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
|
|
1138
|
+
}
|
|
1139
|
+
else {
|
|
1140
|
+
this._log('Generating new grid and placing initial orders on-chain...');
|
|
1141
|
+
await this.placeInitialOrders();
|
|
1142
|
+
}
|
|
1143
|
+
await this._persistAndRecoverIfNeeded();
|
|
1144
|
+
}
|
|
1145
|
+
else {
|
|
1146
|
+
this._log('Found active session. Loading and syncing existing grid.');
|
|
1147
|
+
await Grid.loadGrid(this.manager, persistedGrid, persistedBoundaryIdx);
|
|
1148
|
+
let startupChainOpenOrders = chainOpenOrders;
|
|
1149
|
+
const syncResult = await this.manager.syncFromOpenOrders(startupChainOpenOrders, { skipAccounting: true, fillLockAlreadyHeld: true });
|
|
1150
|
+
if (syncResult.filledOrders && syncResult.filledOrders.length > 0) {
|
|
1151
|
+
this._log(`Startup sync: ${syncResult.filledOrders.length} grid order(s) found filled. Processing proceeds.`, 'info');
|
|
1152
|
+
const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'startup sync fill rebalance', { skipAccountTotalsUpdate: true });
|
|
1153
|
+
if (!batchResult?.aborted) {
|
|
1154
|
+
// Refresh open orders so startup reconcile works with post-batch chain reality
|
|
1155
|
+
// and avoids reconciling against a stale pre-batch snapshot.
|
|
1156
|
+
startupChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1157
|
+
await this.manager.synchronizeWithChain(startupChainOpenOrders, 'readOpenOrders', { fillLockAlreadyHeld: true });
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
const rebalanceResult = await reconcileGridOrders({
|
|
1161
|
+
manager: this.manager,
|
|
1162
|
+
config: this.config,
|
|
1163
|
+
account: this.account,
|
|
1164
|
+
privateKey: this.privateKey,
|
|
1165
|
+
chainOrders,
|
|
1166
|
+
chainOpenOrders: startupChainOpenOrders,
|
|
1167
|
+
fillLockAlreadyHeld: true,
|
|
1168
|
+
});
|
|
1169
|
+
await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (loaded grid)');
|
|
1170
|
+
// Dust state is no longer persisted — cancelled immediately on detection.
|
|
1171
|
+
await this._persistAndRecoverIfNeeded();
|
|
1172
|
+
}
|
|
1173
|
+
// Drain any fills that arrived during startup while still in bootstrap
|
|
1174
|
+
// mode. Safe to call directly since we already hold _fillProcessingLock
|
|
1175
|
+
// and _processFillsWithBootstrapMode does NOT re-acquire it.
|
|
1176
|
+
if (this._incomingFillQueue.length > 0) {
|
|
1177
|
+
this._log(`[STARTUP] Processing ${this._incomingFillQueue.length} queued fill(s) before bootstrap ends`);
|
|
1178
|
+
await this._processFillsWithBootstrapMode(chainOrders);
|
|
1179
|
+
}
|
|
1180
|
+
this.manager.finishBootstrap();
|
|
1181
|
+
// Perform initial grid maintenance (thresholds, divergence, spread, health)
|
|
1182
|
+
// Consolidated into shared logic to ensure consistent behavior at boot and runtime.
|
|
1183
|
+
// CRITICAL: Pass lockAlreadyHeld since we're inside _fillProcessingLock.acquire()
|
|
1184
|
+
await this._runGridMaintenance('startup', { fillLockAlreadyHeld: true });
|
|
1185
|
+
// Cancel any dust from a previous bot lifetime immediately.
|
|
1186
|
+
const startupHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
1187
|
+
await this._cancelDustOrders({
|
|
1188
|
+
buy: startupHealth.buyDustOrders,
|
|
1189
|
+
sell: startupHealth.sellDustOrders,
|
|
1190
|
+
fillLockAlreadyHeld: true,
|
|
1191
|
+
});
|
|
1192
|
+
this._log('Bootstrap phase complete - fill processing resumed', 'info');
|
|
1193
|
+
}
|
|
1194
|
+
finally {
|
|
1195
|
+
// CRITICAL: Always clear bootstrap flag, even on error
|
|
1196
|
+
this.manager.finishBootstrap();
|
|
1197
|
+
}
|
|
1198
|
+
});
|
|
1199
|
+
await this._setupTriggerFileDetection();
|
|
1200
|
+
await this._setupCreditRuntime();
|
|
1201
|
+
await this._refreshAndSyncCreditRuntime();
|
|
1202
|
+
await this._runCreditRuntimeMaintenance('startup', { fillLockAlreadyHeld: true });
|
|
1203
|
+
this._setupBlockchainFetchInterval();
|
|
1204
|
+
this._setupCreditWatchdogInterval();
|
|
1205
|
+
this._setupCredentialDaemonWatchdogInterval();
|
|
1206
|
+
// Periodic dust health check — catches partials that fell below threshold
|
|
1207
|
+
// without triggering the post-fill pipeline. Cancels immediately inside the
|
|
1208
|
+
// fill-processing lock to avoid racing with fill batches or sync operations.
|
|
1209
|
+
this._setupDustHealthCheckInterval();
|
|
1210
|
+
if (this._isOpenOrdersSyncLoopEnabled()) {
|
|
1211
|
+
this._startOpenOrdersSyncLoop();
|
|
1212
|
+
}
|
|
1213
|
+
else {
|
|
1214
|
+
this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
|
|
1215
|
+
}
|
|
1216
|
+
this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
|
|
1217
|
+
}
|
|
1218
|
+
catch (err) {
|
|
1219
|
+
this._warn(`Error during grid initialization: ${err.message}`);
|
|
1220
|
+
await this.shutdown();
|
|
1221
|
+
throw err;
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
/**
|
|
1225
|
+
* Create the fill callback for listenForFills.
|
|
1226
|
+
* Separated from start() to allow deferred activation after startup completes.
|
|
1227
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
1228
|
+
* @returns {Function} Async callback for processing fills
|
|
1229
|
+
* @private
|
|
1230
|
+
*/
|
|
1231
|
+
_createFillCallback(chainOrders) {
|
|
1232
|
+
return DexbotFillRuntime.createFillCallback(this, chainOrders);
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Read open orders from chain, sync with local state, and process any fills found.
|
|
1236
|
+
* Shared helper used by post-reset spread check and targeted drift reconciliation.
|
|
1237
|
+
* @param {string} tag - Context label for logging
|
|
1238
|
+
* @returns {Promise<{syncResult: Object|null, aborted: boolean, hasUnmatched: number, openOrders: Array|null}>}
|
|
1239
|
+
*/
|
|
1240
|
+
async _syncOpenOrdersAndProcessFills(tag) {
|
|
1241
|
+
if (!this.accountId || this.config?.dryRun) {
|
|
1242
|
+
return { syncResult: null, aborted: false, hasUnmatched: 0, openOrders: null };
|
|
1243
|
+
}
|
|
1244
|
+
try {
|
|
1245
|
+
let openOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1246
|
+
const syncResult = await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders', { fillLockAlreadyHeld: true });
|
|
1247
|
+
let aborted = false;
|
|
1248
|
+
if (syncResult?.filledOrders?.length > 0) {
|
|
1249
|
+
this._log(`[SYNC-CHAIN] ${syncResult.filledOrders.length} filled order(s) found during ${tag}`, 'info');
|
|
1250
|
+
const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), `${tag} sync-fill`);
|
|
1251
|
+
if (!batchResult?.aborted) {
|
|
1252
|
+
openOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1253
|
+
await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders', { fillLockAlreadyHeld: true });
|
|
1254
|
+
}
|
|
1255
|
+
else {
|
|
1256
|
+
aborted = true;
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
const hasUnmatched = syncResult?.unmatchedChainOrders?.length || 0;
|
|
1260
|
+
return { syncResult, aborted, hasUnmatched, openOrders };
|
|
1261
|
+
}
|
|
1262
|
+
catch (err) {
|
|
1263
|
+
this._warn(`[SYNC-CHAIN] Open-orders sync failed during ${tag}: ${err.message}`);
|
|
1264
|
+
return { syncResult: null, aborted: true, hasUnmatched: -1, openOrders: null };
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
_maxConsecutiveFillConsumerFailures() {
|
|
1268
|
+
return FILL_PROCESSING.MAX_CONSECUTIVE_CONSUMER_FAILURES;
|
|
1269
|
+
}
|
|
1270
|
+
/**
|
|
1271
|
+
* Compute the backoff delay for fill-consumer retries after the failure
|
|
1272
|
+
* budget (MAX_CONSECUTIVE_CONSUMER_FAILURES) is exhausted. Each retry
|
|
1273
|
+
* doubles the previous delay, capped at CONSUMER_BACKOFF_MAX_MS. The
|
|
1274
|
+
* consumer NEVER permanently stops re-scheduling — it just slows down.
|
|
1275
|
+
* @param {number} failures The current consecutive-failure count.
|
|
1276
|
+
* @returns {number} Delay in milliseconds before the next retry.
|
|
1277
|
+
* @private
|
|
1278
|
+
*/
|
|
1279
|
+
_computeFillConsumerBackoffMs(failures) {
|
|
1280
|
+
const initial = FILL_PROCESSING.CONSUMER_BACKOFF_INITIAL_MS;
|
|
1281
|
+
const max = FILL_PROCESSING.CONSUMER_BACKOFF_MAX_MS;
|
|
1282
|
+
const stepAfterMax = Math.max(0, failures - FILL_PROCESSING.MAX_CONSECUTIVE_CONSUMER_FAILURES);
|
|
1283
|
+
// 0 -> initial, 1 -> 2*initial, 2 -> 4*initial, ... capped at max.
|
|
1284
|
+
return Math.min(max, initial * Math.pow(2, stepAfterMax));
|
|
1285
|
+
}
|
|
1286
|
+
_scheduleFillConsumerRestart(chainOrders) {
|
|
1287
|
+
const failures = this._consecutiveConsumeFailures;
|
|
1288
|
+
if (failures >= FILL_PROCESSING.MAX_CONSECUTIVE_CONSUMER_FAILURES) {
|
|
1289
|
+
// Past the failure budget: switch from tight setImmediate loop to
|
|
1290
|
+
// exponential backoff. The consumer continues to retry — a transient
|
|
1291
|
+
// outage (e.g., credential daemon recovery) will resume normal
|
|
1292
|
+
// operation as soon as one cycle succeeds and resets the counter.
|
|
1293
|
+
// A permanent failure mode yields slower-but-still-progressing
|
|
1294
|
+
// retries capped at CONSUMER_BACKOFF_MAX_MS, with no permanent
|
|
1295
|
+
// stop that would require a bot restart.
|
|
1296
|
+
const backoffMs = this._computeFillConsumerBackoffMs(failures);
|
|
1297
|
+
const elapsedSec = this._consumeFailureFirstAt
|
|
1298
|
+
? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
|
|
1299
|
+
: null;
|
|
1300
|
+
const elapsed = elapsedSec !== null ? `${elapsedSec}s` : 'unknown';
|
|
1301
|
+
// Escalate log level on sustained failure so operators monitoring
|
|
1302
|
+
// for error/critical alerts are not blind to a stuck consumer.
|
|
1303
|
+
// - warn : within the first escalation window (5-9 failures,
|
|
1304
|
+
// <5 min) — could be a slow recovery
|
|
1305
|
+
// - error : 10+ failures OR 5+ minutes of sustained failure
|
|
1306
|
+
// - critical: 20+ failures OR 15+ minutes of sustained failure —
|
|
1307
|
+
// this is the "permanent fault" signal
|
|
1308
|
+
const sustainedLevel = (failures >= 20 || (elapsedSec !== null && elapsedSec >= 900))
|
|
1309
|
+
? 'critical'
|
|
1310
|
+
: (failures >= 10 || (elapsedSec !== null && elapsedSec >= 300))
|
|
1311
|
+
? 'error'
|
|
1312
|
+
: 'warn';
|
|
1313
|
+
this._log(`[FILL-QUEUE] Fill consumer has failed ${failures} consecutive times over ${elapsed}; ` +
|
|
1314
|
+
`backing off ${Math.round(backoffMs / TIMING.MILLISECONDS_PER_SECOND)}s before retry. ` +
|
|
1315
|
+
`Queue: ${this._incomingFillQueue.length} fills.`, sustainedLevel);
|
|
1316
|
+
setTimeout(() => {
|
|
1317
|
+
if (this._shuttingDown)
|
|
1318
|
+
return;
|
|
1319
|
+
this._consumeFillQueue(chainOrders).catch(err => {
|
|
1320
|
+
if (!this._consumeFailureFirstAt) {
|
|
1321
|
+
this._consumeFailureFirstAt = Date.now();
|
|
1322
|
+
}
|
|
1323
|
+
this._consecutiveConsumeFailures++;
|
|
1324
|
+
const newFailures = this._consecutiveConsumeFailures;
|
|
1325
|
+
const newElapsedSec = this._consumeFailureFirstAt
|
|
1326
|
+
? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
|
|
1327
|
+
: null;
|
|
1328
|
+
const resumeLevel = (newFailures >= 20 || (newElapsedSec !== null && newElapsedSec >= 900))
|
|
1329
|
+
? 'critical'
|
|
1330
|
+
: (newFailures >= 10 || (newElapsedSec !== null && newElapsedSec >= 300))
|
|
1331
|
+
? 'error'
|
|
1332
|
+
: 'warn';
|
|
1333
|
+
this._log(`Fill consumer resume after backoff failed ` +
|
|
1334
|
+
`(${newFailures} total, ` +
|
|
1335
|
+
`next backoff ${Math.round(this._computeFillConsumerBackoffMs(newFailures) / TIMING.MILLISECONDS_PER_SECOND)}s): ` +
|
|
1336
|
+
`${err.message}`, resumeLevel);
|
|
1337
|
+
// Continue the backoff loop. The success path of
|
|
1338
|
+
// _consumeFillQueue resets the counter, breaking the cycle.
|
|
1339
|
+
this._scheduleFillConsumerRestart(chainOrders);
|
|
1340
|
+
});
|
|
1341
|
+
}, backoffMs);
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
setImmediate(() => this._consumeFillQueue(chainOrders).catch(err => {
|
|
1345
|
+
if (!this._consumeFailureFirstAt) {
|
|
1346
|
+
this._consumeFailureFirstAt = Date.now();
|
|
1347
|
+
}
|
|
1348
|
+
this._consecutiveConsumeFailures++;
|
|
1349
|
+
const remaining = this._maxConsecutiveFillConsumerFailures() - this._consecutiveConsumeFailures;
|
|
1350
|
+
this._log(`Fill consumer failed (${this._consecutiveConsumeFailures}/${this._maxConsecutiveFillConsumerFailures()}, ` +
|
|
1351
|
+
`${remaining} attempts remaining): ${err.message}`, this._consecutiveConsumeFailures >= 3 ? 'warn' : 'error');
|
|
1352
|
+
}));
|
|
1353
|
+
}
|
|
1354
|
+
/**
|
|
1355
|
+
* Consume queued fills from incomingFillQueue and rebalance.
|
|
1356
|
+
*
|
|
1357
|
+
* 1. Deduplicates fills against already-processed set (replay-safe)
|
|
1358
|
+
* 2. Syncs filled orders from history or open orders mode
|
|
1359
|
+
* 3. Handles price mismatches via correctAllPriceMismatches
|
|
1360
|
+
* 4. Processes fills sequentially with interruptible rebalancing (merges new work between fills)
|
|
1361
|
+
* 5. Periodically cleans old fill records to prevent memory leaks
|
|
1362
|
+
*
|
|
1363
|
+
* Atomic lock behavior: If already processing or has waiters, returns immediately (no double-queuing)
|
|
1364
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
1365
|
+
* @private
|
|
1366
|
+
*/
|
|
1367
|
+
async _consumeFillQueue(chainOrders) {
|
|
1368
|
+
// Helper: every early return below is a "deferral", not a failure.
|
|
1369
|
+
// The counter only tracks actual failures, so any healthy deferral
|
|
1370
|
+
// path should also reset the counter. Without this, a sequence of
|
|
1371
|
+
// F-S-F-S-F-S-F-S-F (fail, succeed, fail, succeed, ...) would still
|
|
1372
|
+
// reach the max and trip backoff, OR a failure followed by an empty
|
|
1373
|
+
// queue / shutdown / in-flight batch would leave the counter sticky
|
|
1374
|
+
// and trigger backoff one step sooner on the next real failure.
|
|
1375
|
+
const resetFailureWatchdogIfSet = () => {
|
|
1376
|
+
if (this._consecutiveConsumeFailures > 0 || this._consumeFailureFirstAt > 0) {
|
|
1377
|
+
this._consecutiveConsumeFailures = 0;
|
|
1378
|
+
this._consumeFailureFirstAt = 0;
|
|
1379
|
+
}
|
|
1380
|
+
};
|
|
1381
|
+
// ATOMIC: Only attempt lock acquisition if queue has work
|
|
1382
|
+
// This prevents unnecessary lock contention on empty queues
|
|
1383
|
+
if (this._incomingFillQueue.length === 0) {
|
|
1384
|
+
// Empty queue = consumer is healthy, just idle.
|
|
1385
|
+
resetFailureWatchdogIfSet();
|
|
1386
|
+
return;
|
|
1387
|
+
}
|
|
1388
|
+
// Check shutdown state
|
|
1389
|
+
if (this._shuttingDown) {
|
|
1390
|
+
this._warn('Fill processing skipped: shutdown in progress');
|
|
1391
|
+
resetFailureWatchdogIfSet();
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
if (this._batchInFlight || this._recoverySyncInFlight) {
|
|
1395
|
+
this.manager?.logger?.log?.(`Fill processing deferred: order pipeline active (${this._incomingFillQueue.length} queued)`, 'debug');
|
|
1396
|
+
// A batch is in flight, not a failure. The next iteration will
|
|
1397
|
+
// either succeed (and reset the counter on the success path) or
|
|
1398
|
+
// fail and increment it. Either way, leaving an old failure
|
|
1399
|
+
// count here would double-count.
|
|
1400
|
+
resetFailureWatchdogIfSet();
|
|
1401
|
+
return;
|
|
1402
|
+
}
|
|
1403
|
+
let pendingFillKeysForCurrentCycle = new Set();
|
|
1404
|
+
try {
|
|
1405
|
+
// BOOTSTRAP OPTIMIZATION: During bootstrap, prioritize fill processing over grid-wide checks
|
|
1406
|
+
// Process fills immediately with side-only rebalancing (no expensive full grid recalculations)
|
|
1407
|
+
if (this.manager._state.isBootstrapping()) {
|
|
1408
|
+
// During bootstrap: skip lock contention checks, process fills directly
|
|
1409
|
+
let bootstrapSkipped = false;
|
|
1410
|
+
await this.manager._fillProcessingLock.acquire(async () => {
|
|
1411
|
+
if (!this.manager._state.isBootstrapping()) {
|
|
1412
|
+
// Bootstrap finished while waiting for the lock — no
|
|
1413
|
+
// work to do, but the iteration is still healthy.
|
|
1414
|
+
bootstrapSkipped = true;
|
|
1415
|
+
return;
|
|
1416
|
+
}
|
|
1417
|
+
await this._processFillsWithBootstrapMode(chainOrders);
|
|
1418
|
+
});
|
|
1419
|
+
if (bootstrapSkipped) {
|
|
1420
|
+
// Bootstrap-mode lock callback returned without doing
|
|
1421
|
+
// work; the .acquire() success path at line ~1941 that
|
|
1422
|
+
// would normally reset the counter is not reached in
|
|
1423
|
+
// this branch. Reset here so a stale counter from a
|
|
1424
|
+
// prior failure doesn't carry over.
|
|
1425
|
+
resetFailureWatchdogIfSet();
|
|
1426
|
+
}
|
|
1427
|
+
return;
|
|
1428
|
+
}
|
|
1429
|
+
// NORMAL MODE: Non-blocking check if lock already has waiters
|
|
1430
|
+
// This prevents unbounded queue growth while still ensuring processing
|
|
1431
|
+
// Note: We DO proceed if lock is held but has no waiters - we'll wait our turn
|
|
1432
|
+
if (this.manager._fillProcessingLock.getQueueLength() > 0) {
|
|
1433
|
+
this._metrics.lockContentionEvents++;
|
|
1434
|
+
// Deferral, not a failure. Reset the watchdog so the next
|
|
1435
|
+
// call (which may now find an empty queue, or process
|
|
1436
|
+
// successfully) doesn't inherit a stale counter.
|
|
1437
|
+
resetFailureWatchdogIfSet();
|
|
1438
|
+
return;
|
|
1439
|
+
}
|
|
1440
|
+
await this.manager._fillProcessingLock.acquire(async () => {
|
|
1441
|
+
while (this._incomingFillQueue.length > 0) {
|
|
1442
|
+
const batchStartTime = Date.now();
|
|
1443
|
+
// Track max queue depth
|
|
1444
|
+
this._metrics.maxQueueDepth = Math.max(this._metrics.maxQueueDepth, this._incomingFillQueue.length);
|
|
1445
|
+
// 1. Take snapshot of current work (ATOMIC: splice removes and returns fills atomically)
|
|
1446
|
+
const allFills = this._incomingFillQueue.splice(0); // Atomically clear and get all fills
|
|
1447
|
+
const validFills = [];
|
|
1448
|
+
const processedFillKeys = new Set();
|
|
1449
|
+
pendingFillKeysForCurrentCycle = new Set();
|
|
1450
|
+
let requiresOpenOrdersSync = false;
|
|
1451
|
+
// 2. Filter and Deduplicate (Standard Logic)
|
|
1452
|
+
for (const fill of allFills) {
|
|
1453
|
+
if (fill && fill.op && fill.op[0] === FILL_PROCESSING.OPERATION_TYPE) {
|
|
1454
|
+
const fillOp = fill.op[1];
|
|
1455
|
+
// SELF-CANCEL GUARD: only drop malformed, cancel-like
|
|
1456
|
+
// artifacts for an order the local process just cancelled.
|
|
1457
|
+
// Real fill_order ops carry economic data and must still be
|
|
1458
|
+
// accounted even if they arrive shortly after a successful
|
|
1459
|
+
// cancel broadcast.
|
|
1460
|
+
const hasFillEconomics = fillOp?.pays?.asset_id && fillOp?.pays?.amount != null
|
|
1461
|
+
&& fillOp?.receives?.asset_id && fillOp?.receives?.amount != null;
|
|
1462
|
+
if (chainOrders && typeof chainOrders.wasRecentlyOwnCancelled === 'function'
|
|
1463
|
+
&& chainOrders.wasRecentlyOwnCancelled(fillOp.order_id)
|
|
1464
|
+
&& !hasFillEconomics) {
|
|
1465
|
+
this.manager.logger.log(`[SELF-CANCEL] Skipping non-economic fill artifact for order ${fillOp.order_id} (just cancelled by this bot)`, 'debug');
|
|
1466
|
+
continue;
|
|
1467
|
+
}
|
|
1468
|
+
// ACCOUNT VALIDATION: Verify the filled order belongs to this bot's account/grid
|
|
1469
|
+
// Only process fills for orders we actually manage
|
|
1470
|
+
const gridOrder = this.manager.orders.get(fillOp.order_id) ||
|
|
1471
|
+
Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
|
|
1472
|
+
if (!gridOrder) {
|
|
1473
|
+
// Check if this order was already freed by stale-order batch cleanup.
|
|
1474
|
+
// When a batch fails due to a stale order reference, the cleanup converts the
|
|
1475
|
+
// slot to VIRTUAL/SPREAD, releasing committed funds to chainFree. If we also
|
|
1476
|
+
// credit the fill proceeds here, we double-count the capital.
|
|
1477
|
+
const staleMarkedAt = this._staleCleanedOrderIds.get(fillOp.order_id);
|
|
1478
|
+
if (staleMarkedAt != null) {
|
|
1479
|
+
const staleAgeMs = Date.now() - staleMarkedAt;
|
|
1480
|
+
if (staleAgeMs <= this._staleCleanupRetentionMs) {
|
|
1481
|
+
this.manager.logger.log(`[ORPHAN-FILL] Skipping double-credit for stale-cleaned order ${fillOp.order_id} ` +
|
|
1482
|
+
`(funds already freed by batch cleanup, age=${staleAgeMs}ms)`, 'warn');
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1485
|
+
this._staleCleanedOrderIds.delete(fillOp.order_id);
|
|
1486
|
+
}
|
|
1487
|
+
// Legitimate orphan fill: order was virtualized during sequential processing
|
|
1488
|
+
// but a fill arrived afterward. Credit proceeds to maintain fund tracking.
|
|
1489
|
+
let orphanFillKey = buildFillKey(fill);
|
|
1490
|
+
if (!orphanFillKey) {
|
|
1491
|
+
orphanFillKey = this._buildOrphanFillFallbackKey(fill);
|
|
1492
|
+
}
|
|
1493
|
+
if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[ORPHAN-FILL]')) {
|
|
1494
|
+
continue;
|
|
1495
|
+
}
|
|
1496
|
+
this.manager.logger.log(`[ORPHAN-FILL] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
|
|
1497
|
+
const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
|
|
1498
|
+
context: 'ORPHAN-FILL',
|
|
1499
|
+
replayMessage: (op) => `[ORPHAN-FILL] Replay detected for ${op.order_id}; skipping duplicate credit`
|
|
1500
|
+
});
|
|
1501
|
+
if (accountingResult.status === 'missing_key') {
|
|
1502
|
+
requiresOpenOrdersSync = true;
|
|
1503
|
+
}
|
|
1504
|
+
// Don't add to validFills - we can't do rebalancing without a grid slot
|
|
1505
|
+
// But the funds are now credited, preventing fund invariant violation
|
|
1506
|
+
continue;
|
|
1507
|
+
}
|
|
1508
|
+
// Process both maker and taker fills for our grid orders
|
|
1509
|
+
// Grid validation ensures we only process fills belonging to our account
|
|
1510
|
+
// Taker fills are included because the bot may execute market orders or act as taker
|
|
1511
|
+
const roleStr = fillOp.is_maker ? 'maker' : 'taker';
|
|
1512
|
+
this.manager.logger.log(`Processing ${roleStr} fill for order ${fillOp.order_id}`, 'debug');
|
|
1513
|
+
const fillKey = buildFillKey(fill);
|
|
1514
|
+
if (!fillKey) {
|
|
1515
|
+
this.manager.logger.log(`[FILL] Missing history id for order ${fillOp.order_id} block ${fill.block_num}; deferring to open-orders sync`, 'warn');
|
|
1516
|
+
requiresOpenOrdersSync = true;
|
|
1517
|
+
continue;
|
|
1518
|
+
}
|
|
1519
|
+
if (!this._isNewFillKey(fillKey, processedFillKeys, '[FILL]')) {
|
|
1520
|
+
continue;
|
|
1521
|
+
}
|
|
1522
|
+
validFills.push(fill);
|
|
1523
|
+
// Log info
|
|
1524
|
+
const paysAmount = fillOp.pays ? fillOp.pays.amount : '?';
|
|
1525
|
+
const receivesAmount = fillOp.receives ? fillOp.receives.amount : '?';
|
|
1526
|
+
this._log(`\n===== FILL DETECTED =====`);
|
|
1527
|
+
this._log(`Order ID: ${fillOp.order_id}`);
|
|
1528
|
+
this._log(`Pays: ${paysAmount}, Receives: ${receivesAmount}`);
|
|
1529
|
+
this._log(`Block: ${fill.block_num} (History ID: ${fill.id || 'N/A'})`);
|
|
1530
|
+
this._log(`=========================\n`);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
// Clean up short-lived queue dedupe cache to prevent memory leak.
|
|
1534
|
+
const cleanupTimestamp = Date.now();
|
|
1535
|
+
let cleanedCount = 0;
|
|
1536
|
+
for (const [key, timestamp] of this._recentlyQueuedFills) {
|
|
1537
|
+
if (cleanupTimestamp - timestamp > this._fillDedupeWindowMs) {
|
|
1538
|
+
this._recentlyQueuedFills.delete(key);
|
|
1539
|
+
cleanedCount++;
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
if (cleanedCount > 0) {
|
|
1543
|
+
this.manager.logger.log(`Cleaned ${cleanedCount} old queued fill records. Remaining: ${this._recentlyQueuedFills.size}`, 'debug');
|
|
1544
|
+
}
|
|
1545
|
+
if (validFills.length === 0 && !requiresOpenOrdersSync)
|
|
1546
|
+
continue; // Loop back for more
|
|
1547
|
+
// 3. Sync and Collect Filled Orders
|
|
1548
|
+
let allFilledOrders = [];
|
|
1549
|
+
let ordersNeedingCorrection = [];
|
|
1550
|
+
const fillMode = chainOrders.getFillProcessingMode();
|
|
1551
|
+
const processValidFills = async (fillsToSync) => {
|
|
1552
|
+
let resolvedOrders = [];
|
|
1553
|
+
if (fillMode === 'history') {
|
|
1554
|
+
this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (history mode)`, 'info');
|
|
1555
|
+
for (const fill of fillsToSync) {
|
|
1556
|
+
const resultHistory = await this.manager.syncFromFillHistory(fill, {
|
|
1557
|
+
persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.IMMEDIATE
|
|
1558
|
+
});
|
|
1559
|
+
const fillKey = buildFillKey({
|
|
1560
|
+
orderId: fill?.op?.[1]?.order_id,
|
|
1561
|
+
blockNum: fill?.block_num,
|
|
1562
|
+
historyId: fill?.id
|
|
1563
|
+
});
|
|
1564
|
+
if (fillKey)
|
|
1565
|
+
pendingFillKeysForCurrentCycle.add(fillKey);
|
|
1566
|
+
// Dust is handled by post-fill detection below.
|
|
1567
|
+
if (resultHistory.filledOrders)
|
|
1568
|
+
resolvedOrders.push(...resultHistory.filledOrders);
|
|
1569
|
+
if (resultHistory.requiresOpenOrdersSync)
|
|
1570
|
+
requiresOpenOrdersSync = true;
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
if (fillMode !== 'history' || requiresOpenOrdersSync) {
|
|
1574
|
+
if (fillMode === 'history' && requiresOpenOrdersSync) {
|
|
1575
|
+
this.manager.logger.log('Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
|
|
1576
|
+
}
|
|
1577
|
+
this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (open orders mode)`, 'info');
|
|
1578
|
+
const chainOpenOrders = await chainOrders.readOpenOrders(this.account);
|
|
1579
|
+
const resultOpenOrders = await this.manager.syncFromOpenOrders(chainOpenOrders, { fillLockAlreadyHeld: true });
|
|
1580
|
+
// Dust is handled by post-fill detection below.
|
|
1581
|
+
if (resultOpenOrders.filledOrders)
|
|
1582
|
+
resolvedOrders.push(...resultOpenOrders.filledOrders);
|
|
1583
|
+
if (resultOpenOrders.ordersNeedingCorrection)
|
|
1584
|
+
ordersNeedingCorrection.push(...resultOpenOrders.ordersNeedingCorrection);
|
|
1585
|
+
}
|
|
1586
|
+
return resolvedOrders;
|
|
1587
|
+
};
|
|
1588
|
+
this.manager.pauseFundRecalc();
|
|
1589
|
+
try {
|
|
1590
|
+
allFilledOrders = await processValidFills(validFills);
|
|
1591
|
+
// 4. Handle Price Corrections
|
|
1592
|
+
if (ordersNeedingCorrection.length > 0) {
|
|
1593
|
+
const correctionResult = await correctAllPriceMismatches(this.manager, this.account, this.privateKey, chainOrders);
|
|
1594
|
+
if (correctionResult.failed > 0)
|
|
1595
|
+
this.manager.logger.log(`${correctionResult.failed} corrections failed`, 'error');
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
finally {
|
|
1599
|
+
await this.manager.resumeFundRecalc();
|
|
1600
|
+
}
|
|
1601
|
+
// 5. Fixed-Cap Fill Rebalance
|
|
1602
|
+
// - 1..MAX_FILL_BATCH_SIZE fills: unified full-set planning
|
|
1603
|
+
// - larger bursts: fixed-size chunking at MAX_FILL_BATCH_SIZE
|
|
1604
|
+
if (allFilledOrders.length > 0) {
|
|
1605
|
+
const result = await this._processFillsWithBatching(allFilledOrders, null, 'fill set');
|
|
1606
|
+
let abortedFillCycle = result.aborted;
|
|
1607
|
+
if (!abortedFillCycle) {
|
|
1608
|
+
const batchFillKeys = new Set(allFilledOrders.map(filledOrder => buildFillKey({
|
|
1609
|
+
orderId: filledOrder?.orderId,
|
|
1610
|
+
blockNum: filledOrder?.blockNum,
|
|
1611
|
+
historyId: filledOrder?.historyId
|
|
1612
|
+
})).filter(Boolean));
|
|
1613
|
+
await this._flushProcessedFillPersistenceForKeys(batchFillKeys, 'fill-batch-committed');
|
|
1614
|
+
}
|
|
1615
|
+
else {
|
|
1616
|
+
this.manager.logger.log('[FILL-DEDUP] Fill cycle aborted; fill key persistence guarded under abort path.', 'warn');
|
|
1617
|
+
}
|
|
1618
|
+
// 6. Rebalance Recovery Loop (Sequential Extensions)
|
|
1619
|
+
// DISABLED FOR SEQUENTIAL: Each sequential fill already triggers a full rebalance with proper
|
|
1620
|
+
// boundary shift. An additional recovery loop with EMPTY fills causes the boundary to remain
|
|
1621
|
+
// at the last fill's position, leading to wrong operation types (updates instead of rotations)
|
|
1622
|
+
// and operations on the wrong side.
|
|
1623
|
+
//
|
|
1624
|
+
// In the future, recovery loop can be re-enabled for single fills if needed, but ONLY
|
|
1625
|
+
// if it passes the actual fills to processFilledOrders so the boundary shifts correctly.
|
|
1626
|
+
// For now: Each fill = full rebalance with boundary shift = complete correction in one pass.
|
|
1627
|
+
// CRITICAL: Do NOT run spread correction here during sequential fill processing.
|
|
1628
|
+
// The rebalance from each fill should maintain spread naturally. Running spread correction
|
|
1629
|
+
// immediately after creates new orders that may get filled by market before next cycle,
|
|
1630
|
+
// causing cascading fills and potentially SPREAD slots becoming PARTIAL (error condition).
|
|
1631
|
+
// Spread correction runs in the main loop instead.
|
|
1632
|
+
const fullFillCount = allFilledOrders.filter(o => o && o.isPartial !== true).length;
|
|
1633
|
+
const hasAnyFills = allFilledOrders.some(o => o);
|
|
1634
|
+
const shouldRunPostFillChecks = !abortedFillCycle && fullFillCount > 0;
|
|
1635
|
+
const shouldRunDustDetection = !abortedFillCycle && hasAnyFills;
|
|
1636
|
+
if (shouldRunDustDetection) {
|
|
1637
|
+
const healthResult = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
1638
|
+
const allDust = [
|
|
1639
|
+
...(healthResult.buyDustOrders || []),
|
|
1640
|
+
...(healthResult.sellDustOrders || []),
|
|
1641
|
+
];
|
|
1642
|
+
if (allDust.length > 0) {
|
|
1643
|
+
const dustCancelResult = await this._cancelDustOrders({
|
|
1644
|
+
buy: healthResult.buyDustOrders,
|
|
1645
|
+
sell: healthResult.sellDustOrders,
|
|
1646
|
+
fillLockAlreadyHeld: true,
|
|
1647
|
+
});
|
|
1648
|
+
if (dustCancelResult?.batchResult?.aborted) {
|
|
1649
|
+
abortedFillCycle = true;
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
// Run grid maintenance after fills to rebuild degraded grid.
|
|
1654
|
+
// CRITICAL FIX (commit a946c33): Replaced inline divergence checks with centralized
|
|
1655
|
+
// _runGridMaintenance call to ensure pipeline protection applies consistently.
|
|
1656
|
+
// Before: Divergence checks ran immediately after fills, causing race-to-resize
|
|
1657
|
+
// After: Grid maintenance waits for isPipelineEmpty() before structural changes
|
|
1658
|
+
// Run only when the cycle contains at least one full fill.
|
|
1659
|
+
if (shouldRunPostFillChecks && !abortedFillCycle) {
|
|
1660
|
+
await this._runGridMaintenance('post-fill', { fillLockAlreadyHeld: true });
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
else if (pendingFillKeysForCurrentCycle.size > 0) {
|
|
1664
|
+
await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, 'fill-batch-no-rotations');
|
|
1665
|
+
}
|
|
1666
|
+
await retryPersistenceIfNeeded(this.manager);
|
|
1667
|
+
// Periodically clean up old fill records after processing N fills.
|
|
1668
|
+
// Counter is protected by _fillProcessingLock during fill consumption.
|
|
1669
|
+
this._fillCleanupCounter += validFills.length;
|
|
1670
|
+
const cleanupThreshold = MAINTENANCE.CLEANUP_PROBABILITY > 0 && MAINTENANCE.CLEANUP_PROBABILITY < 1
|
|
1671
|
+
? Math.floor(1 / MAINTENANCE.CLEANUP_PROBABILITY)
|
|
1672
|
+
: 100; // Default: every 100 fills
|
|
1673
|
+
if (this._fillCleanupCounter >= cleanupThreshold) {
|
|
1674
|
+
try {
|
|
1675
|
+
await this.accountOrders.cleanOldProcessedFills(TIMING.FILL_RECORD_RETENTION_MS);
|
|
1676
|
+
this._fillCleanupCounter = 0; // Reset counter after cleanup (success or retry on next batch if failed)
|
|
1677
|
+
}
|
|
1678
|
+
catch (err) {
|
|
1679
|
+
this.manager?.logger?.log(`Warning: Fill cleanup failed (will retry): ${err.message}`, 'warn');
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
// Update metrics
|
|
1683
|
+
this._metrics.fillsProcessed += validFills.length;
|
|
1684
|
+
this._metrics.fillProcessingTimeMs += Date.now() - batchStartTime;
|
|
1685
|
+
// Prune expired stale-cleaned order IDs after each processing cycle.
|
|
1686
|
+
if (this._staleCleanedOrderIds.size > 0) {
|
|
1687
|
+
const now = Date.now();
|
|
1688
|
+
let prunedCount = 0;
|
|
1689
|
+
for (const [orderId, markedAt] of this._staleCleanedOrderIds) {
|
|
1690
|
+
if (now - markedAt > this._staleCleanupRetentionMs) {
|
|
1691
|
+
this._staleCleanedOrderIds.delete(orderId);
|
|
1692
|
+
prunedCount++;
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
if (prunedCount > 0) {
|
|
1696
|
+
this.manager.logger.log(`[STALE-CLEANUP] Pruned ${prunedCount} expired stale-cleaned order IDs ` +
|
|
1697
|
+
`(retention=${this._staleCleanupRetentionMs}ms, remaining=${this._staleCleanedOrderIds.size})`, 'debug');
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
} // End while(_incomingFillQueue)
|
|
1701
|
+
this._markGridActivity('fill processing end');
|
|
1702
|
+
// Reset the fill-consumer watchdog on the success path. The
|
|
1703
|
+
// counter is only incremented by _scheduleFillConsumerRestart's
|
|
1704
|
+
// catch handler; without this reset, a fill pattern of
|
|
1705
|
+
// F-S-F-S-F-S-F-S-F would still reach the max and the consumer
|
|
1706
|
+
// would stop being re-scheduled.
|
|
1707
|
+
this._consecutiveConsumeFailures = 0;
|
|
1708
|
+
this._consumeFailureFirstAt = 0;
|
|
1709
|
+
});
|
|
1710
|
+
}
|
|
1711
|
+
catch (err) {
|
|
1712
|
+
const isCredentialOutage = this._isCredentialDaemonError(err);
|
|
1713
|
+
if (pendingFillKeysForCurrentCycle.size > 0) {
|
|
1714
|
+
const flushReason = isCredentialOutage
|
|
1715
|
+
? 'credential-outage-verified-fills'
|
|
1716
|
+
: 'fill-cycle-error-verified-fills';
|
|
1717
|
+
if (isCredentialOutage) {
|
|
1718
|
+
this._credentialRecoveryNeeded = true;
|
|
1719
|
+
this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
|
|
1720
|
+
}
|
|
1721
|
+
try {
|
|
1722
|
+
await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, flushReason, { throwOnError: true });
|
|
1723
|
+
const credentialSuffix = isCredentialOutage
|
|
1724
|
+
? '; grid persistence is suspended until recovery'
|
|
1725
|
+
: '';
|
|
1726
|
+
this.manager?.logger?.log?.(`[FILL-DEDUP] Persisted ${pendingFillKeysForCurrentCycle.size} verified processed-fill write(s) after fill cycle error${credentialSuffix}.`, isCredentialOutage ? 'warn' : 'info');
|
|
1727
|
+
}
|
|
1728
|
+
catch (flushErr) {
|
|
1729
|
+
this.manager?.logger?.log?.(`[FILL-DEDUP] Failed to persist verified fill keys during fill error handling: ${flushErr.message}`, 'warn');
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
if (isCredentialOutage && pendingFillKeysForCurrentCycle.size === 0) {
|
|
1733
|
+
this._credentialRecoveryNeeded = true;
|
|
1734
|
+
this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
|
|
1735
|
+
}
|
|
1736
|
+
this._log(`Error processing fills: ${err.message}`, 'error');
|
|
1737
|
+
if (err.stack)
|
|
1738
|
+
this._log(err.stack, 'error');
|
|
1739
|
+
}
|
|
1740
|
+
// Post-processing: If new fills arrived while processing, schedule another cycle
|
|
1741
|
+
// SAFE: Done outside lock context, no async work in finally block
|
|
1742
|
+
if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
|
|
1743
|
+
this._scheduleFillConsumerRestart(chainOrders);
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
/**
|
|
1747
|
+
* Process fills during bootstrap phase using the standard fill pipeline.
|
|
1748
|
+
*
|
|
1749
|
+
* BOOTSTRAP MODE STRATEGY:
|
|
1750
|
+
* - Delegate to the same fill pipeline as the post-reset path
|
|
1751
|
+
* - Same-side replacement at the filled slot (handled by processFillsOnly)
|
|
1752
|
+
* - Symmetric grid regen via COW rebalance (calculateTargetGrid + reconcileGrid)
|
|
1753
|
+
* - Dust orders are cancelled by the rebalance's isCreateHealthy / cancelSurpluses
|
|
1754
|
+
*
|
|
1755
|
+
* This ensures:
|
|
1756
|
+
* - Identical behavior between bootstrap and post-reset
|
|
1757
|
+
* - Grid symmetry maintained by the rebalance, not by manual rotation
|
|
1758
|
+
* - Budget safety enforced by validateWorkingGridFunds in the COW engine
|
|
1759
|
+
*
|
|
1760
|
+
* @param {Object} chainOrders - Chain orders instance for broadcasting
|
|
1761
|
+
* @returns {Promise<void>}
|
|
1762
|
+
*/
|
|
1763
|
+
async _processFillsWithBootstrapMode(chainOrders) {
|
|
1764
|
+
if (this._incomingFillQueue.length === 0)
|
|
1765
|
+
return;
|
|
1766
|
+
const startTime = Date.now();
|
|
1767
|
+
const fills = this._incomingFillQueue.splice(0);
|
|
1768
|
+
const validFills = [];
|
|
1769
|
+
const processedFillKeys = new Set();
|
|
1770
|
+
let requiresOpenOrdersSync = false;
|
|
1771
|
+
// 1. Validate and deduplicate fills
|
|
1772
|
+
for (const fill of fills) {
|
|
1773
|
+
if (!fill || fill.op?.[0] !== 4)
|
|
1774
|
+
continue;
|
|
1775
|
+
const fillOp = fill.op[1];
|
|
1776
|
+
const gridOrder = this.manager.orders.get(fillOp.order_id) ||
|
|
1777
|
+
Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
|
|
1778
|
+
if (!gridOrder) {
|
|
1779
|
+
// CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
|
|
1780
|
+
// This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
|
|
1781
|
+
let orphanFillKey = buildFillKey(fill);
|
|
1782
|
+
if (!orphanFillKey) {
|
|
1783
|
+
orphanFillKey = this._buildOrphanFillFallbackKey(fill);
|
|
1784
|
+
}
|
|
1785
|
+
if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[BOOTSTRAP]')) {
|
|
1786
|
+
continue;
|
|
1787
|
+
}
|
|
1788
|
+
this.manager.logger.log(`[BOOTSTRAP] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
|
|
1789
|
+
const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
|
|
1790
|
+
context: 'BOOTSTRAP'
|
|
1791
|
+
});
|
|
1792
|
+
if (accountingResult.status === 'missing_key') {
|
|
1793
|
+
requiresOpenOrdersSync = true;
|
|
1794
|
+
}
|
|
1795
|
+
continue;
|
|
1796
|
+
}
|
|
1797
|
+
const trackedFillKey = buildFillKey(fill);
|
|
1798
|
+
if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[BOOTSTRAP]')) {
|
|
1799
|
+
continue;
|
|
1800
|
+
}
|
|
1801
|
+
const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
|
|
1802
|
+
context: 'BOOTSTRAP',
|
|
1803
|
+
replayMessage: (op) => `[BOOTSTRAP] Replay detected for ${op.order_id}; skipping duplicate bootstrap rebalance`
|
|
1804
|
+
});
|
|
1805
|
+
if (accountingResult.status === 'missing_key') {
|
|
1806
|
+
requiresOpenOrdersSync = true;
|
|
1807
|
+
continue;
|
|
1808
|
+
}
|
|
1809
|
+
if (accountingResult.status !== 'applied') {
|
|
1810
|
+
continue;
|
|
1811
|
+
}
|
|
1812
|
+
validFills.push({ ...fill, gridOrder });
|
|
1813
|
+
const fillType = gridOrder.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL';
|
|
1814
|
+
this._log(`[BOOTSTRAP] Fill detected: ${fillType} order (${fillOp.is_maker ? 'maker' : 'taker'})`);
|
|
1815
|
+
}
|
|
1816
|
+
if (requiresOpenOrdersSync) {
|
|
1817
|
+
this._log('[BOOTSTRAP] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
|
|
1818
|
+
const bootstrapChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1819
|
+
const syncResult = await this.manager.syncFromOpenOrders(bootstrapChainOpenOrders, { fillLockAlreadyHeld: true });
|
|
1820
|
+
if (syncResult.filledOrders?.length > 0) {
|
|
1821
|
+
const queuedOrderIds = new Set(validFills.map(fill => fill?.gridOrder?.orderId).filter(Boolean));
|
|
1822
|
+
for (const filledOrder of syncResult.filledOrders) {
|
|
1823
|
+
if (!filledOrder?.orderId || queuedOrderIds.has(filledOrder.orderId))
|
|
1824
|
+
continue;
|
|
1825
|
+
validFills.push({ gridOrder: filledOrder });
|
|
1826
|
+
queuedOrderIds.add(filledOrder.orderId);
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
await this._flushProcessedFillPersistence('bootstrap-batch');
|
|
1831
|
+
if (validFills.length === 0)
|
|
1832
|
+
return;
|
|
1833
|
+
// 2. Process fills through the standard fill pipeline.
|
|
1834
|
+
// This handles same-side replacement, symmetric rebalance, and dust cleanup
|
|
1835
|
+
// via the same path used by the post-reset flow.
|
|
1836
|
+
try {
|
|
1837
|
+
this._log(`[BOOTSTRAP] Processing ${validFills.length} fill(s) through standard pipeline`, 'info');
|
|
1838
|
+
const filledOrders = validFills.map(f => f.gridOrder);
|
|
1839
|
+
const result = await this._processFillsWithBatching(filledOrders, new Set(), '[BOOTSTRAP] fill processing');
|
|
1840
|
+
if (result.aborted) {
|
|
1841
|
+
this._warn('[BOOTSTRAP] Aborted batch due to illegal state; skipping grid persistence this cycle');
|
|
1842
|
+
}
|
|
1843
|
+
this._metrics.fillsProcessed += validFills.length;
|
|
1844
|
+
this._metrics.fillProcessingTimeMs += Date.now() - startTime;
|
|
1845
|
+
}
|
|
1846
|
+
catch (err) {
|
|
1847
|
+
this._warn(`[BOOTSTRAP] Error processing fills: ${err.message}`);
|
|
1848
|
+
this.manager.logger.log(`[BOOTSTRAP] Fill error: ${err.message}`, 'error');
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
/**
|
|
1852
|
+
* Set up account identifier and configure global context.
|
|
1853
|
+
* @param {string} accountName - The name of the account to set up
|
|
1854
|
+
* @private
|
|
1855
|
+
*/
|
|
1856
|
+
async _setupAccountContext(accountName) {
|
|
1857
|
+
const accId = await chainOrders.resolveAccountId(accountName);
|
|
1858
|
+
if (!accId) {
|
|
1859
|
+
const isIdFormat = /^1\.2\.\d+$/.test(accountName);
|
|
1860
|
+
throw new Error(`Unable to resolve account${isIdFormat ? ' ID' : ''} '${accountName}' on the BitShares blockchain. ` +
|
|
1861
|
+
`Verify the account ${isIdFormat ? 'ID is correct' : 'name is registered and active on chain'}.`);
|
|
1862
|
+
}
|
|
1863
|
+
await chainOrders.setPreferredAccount(accId, accountName);
|
|
1864
|
+
this.account = accountName;
|
|
1865
|
+
this.accountId = accId;
|
|
1866
|
+
this._log(`Initialized DEXBot for account: ${this.account}`);
|
|
1867
|
+
}
|
|
1868
|
+
/**
|
|
1869
|
+
* Initialize the bot by connecting to BitShares and setting up the account.
|
|
1870
|
+
* @param {string|Object|Buffer} [vaultSecret=null] - The unlock secret for authentication.
|
|
1871
|
+
* @returns {Promise<void>}
|
|
1872
|
+
* @throws {Error} If initialization fails or preferredAccount is missing.
|
|
1873
|
+
*/
|
|
1874
|
+
async initialize(vaultSecret = null) {
|
|
1875
|
+
await waitForConnected(TIMING.CONNECTION_TIMEOUT_MS);
|
|
1876
|
+
if (this.config && this.config.preferredAccount) {
|
|
1877
|
+
try {
|
|
1878
|
+
let privateKey = null;
|
|
1879
|
+
try {
|
|
1880
|
+
privateKey = await getKeyStore().resolveSigningKey(this.config.preferredAccount, vaultSecret, BitShares);
|
|
1881
|
+
}
|
|
1882
|
+
catch (err) {
|
|
1883
|
+
if (vaultSecret)
|
|
1884
|
+
throw err;
|
|
1885
|
+
this._warn(`Credential daemon probe failed: ${err.message}. Falling back to interactive authentication.`);
|
|
1886
|
+
}
|
|
1887
|
+
if (!privateKey) {
|
|
1888
|
+
const unlockSecret = await getKeyStore().authenticate();
|
|
1889
|
+
privateKey = await getKeyStore().resolvePrivateKey(this.config.preferredAccount, unlockSecret, BitShares);
|
|
1890
|
+
}
|
|
1891
|
+
this.privateKey = privateKey;
|
|
1892
|
+
await this._setupAccountContext(this.config.preferredAccount);
|
|
1893
|
+
}
|
|
1894
|
+
catch (err) {
|
|
1895
|
+
if (getKeyStore().isMasterPasswordFailure(err)) {
|
|
1896
|
+
throw err;
|
|
1897
|
+
}
|
|
1898
|
+
this._warn(`Auto-selection of preferredAccount failed: ${err.message}`);
|
|
1899
|
+
// dexbot.ts has fallback to selectAccount, bot.ts throws
|
|
1900
|
+
if (typeof chainOrders.selectAccount === 'function') {
|
|
1901
|
+
const accountData = await chainOrders.selectAccount();
|
|
1902
|
+
this.privateKey = accountData.privateKey;
|
|
1903
|
+
await this._setupAccountContext(accountData.accountName);
|
|
1904
|
+
}
|
|
1905
|
+
else {
|
|
1906
|
+
throw err;
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
else {
|
|
1911
|
+
throw new Error('No preferredAccount configured');
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
/**
|
|
1915
|
+
* Places initial orders on the blockchain.
|
|
1916
|
+
* @returns {Promise<void>}
|
|
1917
|
+
*/
|
|
1918
|
+
async placeInitialOrders() {
|
|
1919
|
+
if (!this.manager) {
|
|
1920
|
+
const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
|
|
1921
|
+
this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
|
|
1922
|
+
this.manager.accountOrders = this.accountOrders;
|
|
1923
|
+
}
|
|
1924
|
+
this._wireStructuralGridResyncRequest();
|
|
1925
|
+
this.manager.startBootstrap();
|
|
1926
|
+
try {
|
|
1927
|
+
try {
|
|
1928
|
+
const botFunds = this.config && this.config.botFunds ? this.config.botFunds : {};
|
|
1929
|
+
const needsPercent = (v) => typeof v === 'string' && v.includes('%');
|
|
1930
|
+
if ((needsPercent(botFunds.buy) || needsPercent(botFunds.sell)) && (this.accountId || this.account)) {
|
|
1931
|
+
if (typeof this.manager._fetchAccountBalancesAndSetTotals === 'function') {
|
|
1932
|
+
await this.manager._fetchAccountBalancesAndSetTotals();
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
catch (errFetch) {
|
|
1937
|
+
this._warn(`Could not fetch account totals before initializing grid: ${errFetch && errFetch.message ? errFetch.message : errFetch}`);
|
|
1938
|
+
}
|
|
1939
|
+
await Grid.initializeGrid(this.manager);
|
|
1940
|
+
if (this.config.dryRun) {
|
|
1941
|
+
this.manager.logger.log('Dry run enabled, skipping on-chain order placement.', 'info');
|
|
1942
|
+
await this.manager.persistGrid();
|
|
1943
|
+
return;
|
|
1944
|
+
}
|
|
1945
|
+
this.manager.logger.log('Placing initial orders on-chain...', 'info');
|
|
1946
|
+
const ordersToActivate = this.manager.getInitialOrdersToActivate();
|
|
1947
|
+
const orderGroups = this._buildOutsideInPairGroupsForOrders(ordersToActivate);
|
|
1948
|
+
for (const group of orderGroups) {
|
|
1949
|
+
await this.updateOrdersOnChainPlan({ ordersToPlace: group });
|
|
1950
|
+
}
|
|
1951
|
+
await this.manager.persistGrid();
|
|
1952
|
+
}
|
|
1953
|
+
finally {
|
|
1954
|
+
this.manager.finishBootstrap();
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
/**
|
|
1958
|
+
* Build outside-in pair groups for initial order placement.
|
|
1959
|
+
* @param {Array<Object>} orders - Array of order objects
|
|
1960
|
+
* @returns {Array<Array<Object>>} Grouped order arrays
|
|
1961
|
+
*/
|
|
1962
|
+
_buildOutsideInPairGroupsForOrders(orders) {
|
|
1963
|
+
return buildOutsideInPairGroups(orders, {
|
|
1964
|
+
isValid: Boolean,
|
|
1965
|
+
getType: o => o.type,
|
|
1966
|
+
getPrice: o => o.price,
|
|
1967
|
+
});
|
|
1968
|
+
}
|
|
1969
|
+
/**
|
|
1970
|
+
* Build outside-in pair groups for create entry contexts.
|
|
1971
|
+
* @param {Array<Object>} createEntries - Array of create entry objects with context.order
|
|
1972
|
+
* @returns {Array<Array<Object>>} Grouped entry arrays
|
|
1973
|
+
*/
|
|
1974
|
+
_buildOutsideInPairGroupsForCreateEntries(createEntries) {
|
|
1975
|
+
return buildOutsideInPairGroups(createEntries, {
|
|
1976
|
+
isValid: e => Boolean(e?.context?.order),
|
|
1977
|
+
getType: e => e.context.order.type,
|
|
1978
|
+
getPrice: e => e.context.order.price,
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
/**
|
|
1982
|
+
* Resolve the centralized fill batch cap.
|
|
1983
|
+
* @returns {number} Positive maximum number of fill-driven rotations per broadcast cycle
|
|
1984
|
+
*/
|
|
1985
|
+
_getMaxFillBatchSize() {
|
|
1986
|
+
return Math.max(1, FILL_PROCESSING.MAX_FILL_BATCH_SIZE);
|
|
1987
|
+
}
|
|
1988
|
+
/**
|
|
1989
|
+
* Extract operation results from a batch transaction result.
|
|
1990
|
+
* @param {Object|Array|null} result - Transaction result from executeBatch
|
|
1991
|
+
* @param {string} [warnContext=''] - Context for warning messages
|
|
1992
|
+
* @returns {Array} Array of operation result entries
|
|
1993
|
+
*/
|
|
1994
|
+
_extractOperationResults(result, warnContext = '') {
|
|
1995
|
+
const extracted = extractBatchOperationResults(result);
|
|
1996
|
+
if (Array.isArray(extracted))
|
|
1997
|
+
return extracted;
|
|
1998
|
+
if (result) {
|
|
1999
|
+
const resultType = Array.isArray(result) ? 'array' : typeof result;
|
|
2000
|
+
const keySummary = (resultType === 'object' && !Array.isArray(result))
|
|
2001
|
+
? Object.keys(result).slice(0, 8).join(',')
|
|
2002
|
+
: '';
|
|
2003
|
+
const contextSuffix = warnContext ? ` (${warnContext})` : '';
|
|
2004
|
+
const keysSuffix = keySummary ? `; keys=[${keySummary}]` : '';
|
|
2005
|
+
this.manager?.logger?.log(`[COW] Unrecognized operation_results shape${contextSuffix}; defaulting to empty results. resultType=${resultType}${keysSuffix}`, 'warn');
|
|
2006
|
+
}
|
|
2007
|
+
return [];
|
|
2008
|
+
}
|
|
2009
|
+
/**
|
|
2010
|
+
* Find CREATE operation contexts whose broadcast result did not include a chain order id.
|
|
2011
|
+
*
|
|
2012
|
+
* @param {Array} operationResults - operation_results aligned with opContexts.
|
|
2013
|
+
* @param {Array<Object>} opContexts - Operation context metadata aligned with operations.
|
|
2014
|
+
* @returns {Array<{index:number, ctx:Object}>} Missing create result contexts.
|
|
2015
|
+
*/
|
|
2016
|
+
_findMissingCreateResultContexts(operationResults, opContexts) {
|
|
2017
|
+
const missing = [];
|
|
2018
|
+
if (!Array.isArray(opContexts))
|
|
2019
|
+
return missing;
|
|
2020
|
+
for (let i = 0; i < opContexts.length; i++) {
|
|
2021
|
+
const ctx = opContexts[i];
|
|
2022
|
+
if (ctx?.kind !== 'create')
|
|
2023
|
+
continue;
|
|
2024
|
+
const chainOrderId = operationResults?.[i]?.[1];
|
|
2025
|
+
if (!chainOrderId || !/^1\.7\.\d+$/.test(String(chainOrderId))) {
|
|
2026
|
+
missing.push({ index: i, ctx });
|
|
2027
|
+
}
|
|
2028
|
+
}
|
|
2029
|
+
return missing;
|
|
2030
|
+
}
|
|
2031
|
+
/**
|
|
2032
|
+
* Run an immediate chain sync after a successful CREATE broadcast returned incomplete ids.
|
|
2033
|
+
*
|
|
2034
|
+
* Missing-create blockers are intentionally preserved if the recovery snapshot does not
|
|
2035
|
+
* account for the affected local slot. The sync engine owns normal clearing of
|
|
2036
|
+
* _lastUnmatchedChainOrders after a successful clean snapshot; this method prevents a
|
|
2037
|
+
* lagging empty snapshot from clearing blockers that were just created by this flow.
|
|
2038
|
+
*
|
|
2039
|
+
* @param {string} [reason] - Human-readable recovery context for logs.
|
|
2040
|
+
* @returns {Promise<void>}
|
|
2041
|
+
*/
|
|
2042
|
+
async _recoverAfterMissingCreateResults(reason = 'missing create operation results') {
|
|
2043
|
+
try {
|
|
2044
|
+
const accountRef = this.accountId || this.account?.id || this.account;
|
|
2045
|
+
if (!accountRef || !this.manager || !chainOrders?.readOpenOrders) {
|
|
2046
|
+
this.manager?.logger?.log?.(`[COW] Recovery sync unavailable after ${reason}`, 'warn');
|
|
2047
|
+
return;
|
|
2048
|
+
}
|
|
2049
|
+
const preRecoveryMissingCreateBlockers = Array.isArray(this.manager._lastUnmatchedChainOrders)
|
|
2050
|
+
? this.manager._lastUnmatchedChainOrders
|
|
2051
|
+
.filter(order => order?.reason === 'missing-create-result')
|
|
2052
|
+
.map(order => ({ ...order }))
|
|
2053
|
+
: [];
|
|
2054
|
+
const openOrders = await chainOrders.readOpenOrders(accountRef);
|
|
2055
|
+
const recoveryResult = await this.manager.syncFromOpenOrders(openOrders, {
|
|
2056
|
+
skipAccounting: false,
|
|
2057
|
+
fillLockAlreadyHeld: true
|
|
2058
|
+
});
|
|
2059
|
+
this._preserveMissingCreateBlockersAfterRecovery(preRecoveryMissingCreateBlockers, recoveryResult);
|
|
2060
|
+
// Persist any master grid mutations from the recovery sync. The
|
|
2061
|
+
// caller returned from the COW catch handler before reaching the
|
|
2062
|
+
// success-path persistGrid.
|
|
2063
|
+
if (typeof this.manager.persistGrid === 'function') {
|
|
2064
|
+
await this.manager.persistGrid();
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
catch (err) {
|
|
2068
|
+
this.manager?.logger?.log?.(`[COW] CRITICAL: Recovery sync failed after ${reason}: ${err.message}`, 'error');
|
|
2069
|
+
if (typeof this.manager?.requestStructuralGridResync === 'function') {
|
|
2070
|
+
try {
|
|
2071
|
+
await this.manager.requestStructuralGridResync(`recovery sync failed after ${reason}`, {
|
|
2072
|
+
error: err.message
|
|
2073
|
+
});
|
|
2074
|
+
}
|
|
2075
|
+
catch (scheduleErr) {
|
|
2076
|
+
this.manager?.logger?.log?.(`[COW] CRITICAL: Failed to schedule structural resync after recovery failure: ${scheduleErr.message}`, 'error');
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
/**
|
|
2082
|
+
* Restore unresolved missing-create blockers after recovery if sync did not adopt them.
|
|
2083
|
+
*
|
|
2084
|
+
* @param {Array<Object>} blockers - Pre-recovery missing-create blockers.
|
|
2085
|
+
* @param {Object} recoveryResult - Result returned by manager.syncFromOpenOrders.
|
|
2086
|
+
* @returns {void}
|
|
2087
|
+
*/
|
|
2088
|
+
_preserveMissingCreateBlockersAfterRecovery(blockers, recoveryResult) {
|
|
2089
|
+
if (!Array.isArray(blockers) || blockers.length === 0 || !this.manager)
|
|
2090
|
+
return;
|
|
2091
|
+
const adoptedSlotIds = new Set((Array.isArray(recoveryResult?.updatedOrders) ? recoveryResult.updatedOrders : [])
|
|
2092
|
+
.filter(order => order?.id && order?.orderId)
|
|
2093
|
+
.map(order => order.id));
|
|
2094
|
+
const unresolvedBlockers = blockers.filter(blocker => !blocker.slotId || !adoptedSlotIds.has(blocker.slotId));
|
|
2095
|
+
if (unresolvedBlockers.length === 0)
|
|
2096
|
+
return;
|
|
2097
|
+
const currentUnmatched = Array.isArray(this.manager._lastUnmatchedChainOrders)
|
|
2098
|
+
? this.manager._lastUnmatchedChainOrders
|
|
2099
|
+
: [];
|
|
2100
|
+
const currentKeys = new Set(currentUnmatched.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
|
|
2101
|
+
const restored = [...currentUnmatched];
|
|
2102
|
+
for (const blocker of unresolvedBlockers) {
|
|
2103
|
+
const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
|
|
2104
|
+
if (!currentKeys.has(key))
|
|
2105
|
+
restored.push({ ...blocker });
|
|
2106
|
+
}
|
|
2107
|
+
if (restored.length !== currentUnmatched.length) {
|
|
2108
|
+
this.manager._lastUnmatchedChainOrders = restored;
|
|
2109
|
+
this.manager._lastUnmatchedChainOrdersAt = Date.now();
|
|
2110
|
+
this.manager.logger?.log?.(`[COW] Preserving ${restored.length - currentUnmatched.length} missing-create blocker(s) after recovery sync; ` +
|
|
2111
|
+
`chain snapshot did not account for the affected slot(s).`, 'warn');
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
/**
|
|
2115
|
+
* Merge missing CREATE result contexts into manager._lastUnmatchedChainOrders.
|
|
2116
|
+
*
|
|
2117
|
+
* The sync engine sets and clears _lastUnmatchedChainOrders on full sync snapshots.
|
|
2118
|
+
* COW uses the same manager field as a structural create blocker before broadcasting.
|
|
2119
|
+
* Missing-create entries are keyed by reason:slotId:operationIndex to avoid replacing
|
|
2120
|
+
* unrelated unmatched chain orders that may already be blocking new creates.
|
|
2121
|
+
*
|
|
2122
|
+
* @param {Array<{index:number, ctx:Object}>} missingCreateResults - Missing CREATE results.
|
|
2123
|
+
* @returns {void}
|
|
2124
|
+
*/
|
|
2125
|
+
_markMissingCreateResultsAsStructuralBlocker(missingCreateResults) {
|
|
2126
|
+
const blockers = Array.isArray(missingCreateResults)
|
|
2127
|
+
? missingCreateResults.map(item => {
|
|
2128
|
+
const order = item.ctx?.order || {};
|
|
2129
|
+
const fingerprint = [
|
|
2130
|
+
`type=${order.type || 'unknown'}`,
|
|
2131
|
+
`price=${Format.formatPrice6(order.price)}`,
|
|
2132
|
+
`size=${Format.formatAmount(order.size)}`
|
|
2133
|
+
].join(',');
|
|
2134
|
+
return {
|
|
2135
|
+
chainOrderId: 'unknown',
|
|
2136
|
+
type: order.type || null,
|
|
2137
|
+
price: order.price,
|
|
2138
|
+
size: order.size,
|
|
2139
|
+
slotId: order.id || item.ctx?.id || null,
|
|
2140
|
+
reason: 'missing-create-result',
|
|
2141
|
+
operationIndex: item.index,
|
|
2142
|
+
fingerprint,
|
|
2143
|
+
};
|
|
2144
|
+
})
|
|
2145
|
+
: [];
|
|
2146
|
+
if (this.manager && blockers.length > 0) {
|
|
2147
|
+
const existing = Array.isArray(this.manager._lastUnmatchedChainOrders)
|
|
2148
|
+
? this.manager._lastUnmatchedChainOrders
|
|
2149
|
+
: [];
|
|
2150
|
+
const keys = new Set(existing.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
|
|
2151
|
+
const merged = [...existing];
|
|
2152
|
+
for (const blocker of blockers) {
|
|
2153
|
+
const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
|
|
2154
|
+
if (!keys.has(key)) {
|
|
2155
|
+
merged.push(blocker);
|
|
2156
|
+
keys.add(key);
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
this.manager._lastUnmatchedChainOrders = merged;
|
|
2160
|
+
this.manager._lastUnmatchedChainOrdersAt = Date.now();
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
/**
|
|
2164
|
+
* Format an unmatched chain order/blocker for COW logs.
|
|
2165
|
+
*
|
|
2166
|
+
* @param {Object} order - Unmatched chain order or structural blocker.
|
|
2167
|
+
* @returns {string} Compact human-readable diagnostic.
|
|
2168
|
+
*/
|
|
2169
|
+
_formatUnmatchedChainOrderForLog(order) {
|
|
2170
|
+
return formatUnmatchedChainOrder(order);
|
|
2171
|
+
}
|
|
2172
|
+
/**
|
|
2173
|
+
* Record a pending CREATE broadcast on the manager.
|
|
2174
|
+
*
|
|
2175
|
+
* Called immediately after each CREATE op is built into the opContext list.
|
|
2176
|
+
* The fingerprint and op indices are stashed so the recovery path in
|
|
2177
|
+
* _reconcileAfterUncertainBroadcast can correlate the planned op with an
|
|
2178
|
+
* on-chain order (or discard it as a chain-side orphan).
|
|
2179
|
+
*
|
|
2180
|
+
* Storage: manager._pendingBroadcasts is a Map<fingerprint, PendingEntry>.
|
|
2181
|
+
* We store on the manager (not on the bot) so the sync engine, grid
|
|
2182
|
+
* reconcile, and any other consumer can read it without crossing the
|
|
2183
|
+
* bot/manager boundary.
|
|
2184
|
+
*
|
|
2185
|
+
* @param {Object} entry
|
|
2186
|
+
* @param {number} entry.opIndex - Index into the operations array
|
|
2187
|
+
* @param {number} entry.ctxIndex - Index into opContexts
|
|
2188
|
+
* @param {Object} entry.order - The grid order being broadcast
|
|
2189
|
+
* @param {Object} entry.finalInts - { amountToSell, minToReceive, ... } blockchain integers
|
|
2190
|
+
* @returns {void}
|
|
2191
|
+
*/
|
|
2192
|
+
_recordPendingBroadcast(entry) {
|
|
2193
|
+
if (!this.manager || !entry || !entry.order)
|
|
2194
|
+
return;
|
|
2195
|
+
if (!this.manager._pendingBroadcasts || !(this.manager._pendingBroadcasts instanceof Map)) {
|
|
2196
|
+
this.manager._pendingBroadcasts = new Map();
|
|
2197
|
+
}
|
|
2198
|
+
const fingerprint = buildCreateOpFingerprint({
|
|
2199
|
+
side: entry.order.type,
|
|
2200
|
+
assetA: this.manager?.assets?.assetA?.id,
|
|
2201
|
+
assetB: this.manager?.assets?.assetB?.id,
|
|
2202
|
+
sellInt: entry.finalInts?.sell,
|
|
2203
|
+
receiveInt: entry.finalInts?.receive,
|
|
2204
|
+
slotId: entry.order.id
|
|
2205
|
+
});
|
|
2206
|
+
if (!fingerprint) {
|
|
2207
|
+
this.manager.logger.log?.(`[COW] Skipped pending-broadcast record: could not build fingerprint for ${entry.order?.id || 'unknown'}`, 'warn');
|
|
2208
|
+
return;
|
|
2209
|
+
}
|
|
2210
|
+
this.manager._pendingBroadcasts.set(fingerprint, {
|
|
2211
|
+
fingerprint,
|
|
2212
|
+
opIndex: entry.opIndex,
|
|
2213
|
+
ctxIndex: entry.ctxIndex,
|
|
2214
|
+
slotId: entry.order.id,
|
|
2215
|
+
orderId: entry.order.id,
|
|
2216
|
+
orderType: entry.order.type,
|
|
2217
|
+
order: entry.order,
|
|
2218
|
+
finalInts: entry.finalInts,
|
|
2219
|
+
batchId: this._currentBatchId || null,
|
|
2220
|
+
recordedAt: Date.now()
|
|
2221
|
+
});
|
|
2222
|
+
}
|
|
2223
|
+
/**
|
|
2224
|
+
* Clear the pending-broadcast cache.
|
|
2225
|
+
*
|
|
2226
|
+
* Called after a successful commit, after a confirmed failure (so the
|
|
2227
|
+
* stale entries don't block the next cycle), and after a successful
|
|
2228
|
+
* recovery adoption (matched entries are explicitly removed by
|
|
2229
|
+
* _reconcileAfterUncertainBroadcast before calling this).
|
|
2230
|
+
*/
|
|
2231
|
+
_clearPendingBroadcasts() {
|
|
2232
|
+
if (this.manager && this.manager._pendingBroadcasts instanceof Map) {
|
|
2233
|
+
this.manager._pendingBroadcasts.clear();
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
/**
|
|
2237
|
+
* Build a fingerprint for an on-chain order so it can be matched against
|
|
2238
|
+
* the pending-broadcast cache.
|
|
2239
|
+
*
|
|
2240
|
+
* @param {Object} chainOrder - Parsed chain order (id, sell, receive, sellAssetId, receiveAssetId, ...)
|
|
2241
|
+
* @param {string} slotId - The grid slot id (order.id) we expect this chain order to belong to
|
|
2242
|
+
* @returns {string|null} Fingerprint or null on bad input
|
|
2243
|
+
*/
|
|
2244
|
+
_buildChainOrderFingerprint(chainOrder, slotId) {
|
|
2245
|
+
if (!chainOrder || !slotId)
|
|
2246
|
+
return null;
|
|
2247
|
+
const normalized = this._normalizeChainOrderForPendingMatch(chainOrder);
|
|
2248
|
+
if (!normalized)
|
|
2249
|
+
return null;
|
|
2250
|
+
return buildCreateOpFingerprint({
|
|
2251
|
+
side: normalized.side,
|
|
2252
|
+
assetA: normalized.assetA,
|
|
2253
|
+
assetB: normalized.assetB,
|
|
2254
|
+
sellInt: normalized.sellInt,
|
|
2255
|
+
receiveInt: normalized.receiveInt,
|
|
2256
|
+
slotId
|
|
2257
|
+
});
|
|
2258
|
+
}
|
|
2259
|
+
/**
|
|
2260
|
+
* Normalize raw BitShares limit_order_object data into the integer tuple
|
|
2261
|
+
* used by pending-broadcast recovery.
|
|
2262
|
+
*
|
|
2263
|
+
* readOpenOrders() returns raw orders with sell_price/for_sale, not the
|
|
2264
|
+
* parsed DEXBot fields type/sellInt/receiveInt. Test fixtures may still
|
|
2265
|
+
* pass the parsed shape, so this helper accepts both.
|
|
2266
|
+
*
|
|
2267
|
+
* @param {Object} chainOrder
|
|
2268
|
+
* @returns {{side: string, assetA: string, assetB: string, sellInt: number, receiveInt: number}|null}
|
|
2269
|
+
*/
|
|
2270
|
+
_normalizeChainOrderForPendingMatch(chainOrder) {
|
|
2271
|
+
if (!chainOrder)
|
|
2272
|
+
return null;
|
|
2273
|
+
const assetA = this.manager?.assets?.assetA?.id;
|
|
2274
|
+
const assetB = this.manager?.assets?.assetB?.id;
|
|
2275
|
+
if (!assetA || !assetB)
|
|
2276
|
+
return null;
|
|
2277
|
+
const explicitSide = (chainOrder.type === 'buy' || chainOrder.type === 'sell')
|
|
2278
|
+
? chainOrder.type
|
|
2279
|
+
: null;
|
|
2280
|
+
const explicitSell = chainOrder.sellInt ?? chainOrder.sell;
|
|
2281
|
+
const explicitReceive = chainOrder.receiveInt ?? chainOrder.receive;
|
|
2282
|
+
if (explicitSide && Number.isFinite(Number(explicitSell)) && Number.isFinite(Number(explicitReceive))) {
|
|
2283
|
+
return {
|
|
2284
|
+
side: explicitSide,
|
|
2285
|
+
assetA,
|
|
2286
|
+
assetB,
|
|
2287
|
+
sellInt: Number(explicitSell),
|
|
2288
|
+
receiveInt: Number(explicitReceive)
|
|
2289
|
+
};
|
|
2290
|
+
}
|
|
2291
|
+
const base = chainOrder.sell_price?.base;
|
|
2292
|
+
const quote = chainOrder.sell_price?.quote;
|
|
2293
|
+
if (!base || !quote || !base.asset_id || !quote.asset_id)
|
|
2294
|
+
return null;
|
|
2295
|
+
const baseAmount = Number(base.amount);
|
|
2296
|
+
const quoteAmount = Number(quote.amount);
|
|
2297
|
+
if (!Number.isFinite(baseAmount) || !Number.isFinite(quoteAmount))
|
|
2298
|
+
return null;
|
|
2299
|
+
if (base.asset_id === assetA && quote.asset_id === assetB) {
|
|
2300
|
+
return { side: 'sell', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
|
|
2301
|
+
}
|
|
2302
|
+
if (base.asset_id === assetB && quote.asset_id === assetA) {
|
|
2303
|
+
return { side: 'buy', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
|
|
2304
|
+
}
|
|
2305
|
+
return null;
|
|
2306
|
+
}
|
|
2307
|
+
/**
|
|
2308
|
+
* Find a chain order that matches a planned slot using price+size proximity.
|
|
2309
|
+
*
|
|
2310
|
+
* Fallback for the case where the chain order's integer pair doesn't
|
|
2311
|
+
* bit-match the planned op (e.g. the daemon normalized the minToReceive
|
|
2312
|
+
* by ±1 unit to force an op, or precision rounding changed a value by 1).
|
|
2313
|
+
* For each open chain order we build a fingerprint candidate per known
|
|
2314
|
+
* slot id and accept the first exact match; if none, we look for a near
|
|
2315
|
+
* match by sell+receive integer proximity.
|
|
2316
|
+
*
|
|
2317
|
+
* @param {Array<Object>} chainOrders - Open chain orders for the account
|
|
2318
|
+
* @param {string} slotId - Planned grid slot id
|
|
2319
|
+
* @param {Object} planned - { sell, receive, orderType } integers from the planned op
|
|
2320
|
+
* @returns {Object|null} Matching chain order, or null
|
|
2321
|
+
*/
|
|
2322
|
+
_findChainOrderForSlot(chainOrders, slotId, planned) {
|
|
2323
|
+
if (!Array.isArray(chainOrders) || !slotId)
|
|
2324
|
+
return null;
|
|
2325
|
+
const assetA = this.manager?.assets?.assetA?.id;
|
|
2326
|
+
const assetB = this.manager?.assets?.assetB?.id;
|
|
2327
|
+
if (!assetA || !assetB)
|
|
2328
|
+
return null;
|
|
2329
|
+
// 1. Exact fingerprint match.
|
|
2330
|
+
for (const o of chainOrders) {
|
|
2331
|
+
const fp = this._buildChainOrderFingerprint(o, slotId);
|
|
2332
|
+
if (fp && this.manager._pendingBroadcasts?.has(fp)) {
|
|
2333
|
+
return o;
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
if (!planned || !Number.isFinite(Number(planned.sell)) || !Number.isFinite(Number(planned.receive))) {
|
|
2337
|
+
return null;
|
|
2338
|
+
}
|
|
2339
|
+
// 2. Near match: same side, sell int within 1, receive int within 1% or 2 units.
|
|
2340
|
+
const targetSell = Number(planned.sell);
|
|
2341
|
+
const targetReceive = Number(planned.receive);
|
|
2342
|
+
const plannedSide = planned.orderType ||
|
|
2343
|
+
planned.side ||
|
|
2344
|
+
this.manager._pendingBroadcasts?.get?.(planned.fingerprint)?.orderType ||
|
|
2345
|
+
this.manager.orders.get(slotId)?.type;
|
|
2346
|
+
if (plannedSide !== 'buy' && plannedSide !== 'sell') {
|
|
2347
|
+
return null;
|
|
2348
|
+
}
|
|
2349
|
+
let best = null;
|
|
2350
|
+
let bestDistance = Infinity;
|
|
2351
|
+
for (const o of chainOrders) {
|
|
2352
|
+
const normalized = this._normalizeChainOrderForPendingMatch(o);
|
|
2353
|
+
if (!normalized)
|
|
2354
|
+
continue;
|
|
2355
|
+
if (normalized.side !== plannedSide)
|
|
2356
|
+
continue;
|
|
2357
|
+
const sell = Number(normalized.sellInt);
|
|
2358
|
+
const receive = Number(normalized.receiveInt);
|
|
2359
|
+
if (!Number.isFinite(sell) || !Number.isFinite(receive))
|
|
2360
|
+
continue;
|
|
2361
|
+
const sellDelta = Math.abs(sell - targetSell);
|
|
2362
|
+
const receiveDelta = Math.abs(receive - targetReceive);
|
|
2363
|
+
const receiveTol = Math.max(2, Math.floor(targetReceive * 0.01));
|
|
2364
|
+
if (sellDelta > 1 || receiveDelta > receiveTol)
|
|
2365
|
+
continue;
|
|
2366
|
+
const distance = sellDelta * 1000 + receiveDelta;
|
|
2367
|
+
if (distance < bestDistance) {
|
|
2368
|
+
best = o;
|
|
2369
|
+
bestDistance = distance;
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
return best;
|
|
2373
|
+
}
|
|
2374
|
+
/**
|
|
2375
|
+
* Reconcile a broadcast whose chain state is unknown.
|
|
2376
|
+
*
|
|
2377
|
+
* Triggered when the credential daemon times out (or hits its inner
|
|
2378
|
+
* deadline) before confirming the broadcast. The chain may or may not
|
|
2379
|
+
* have accepted the operations; we MUST treat the state as uncertain
|
|
2380
|
+
* and recover deterministically.
|
|
2381
|
+
*
|
|
2382
|
+
* Algorithm:
|
|
2383
|
+
* 1. Read the account's current open orders from the chain.
|
|
2384
|
+
* 2. For each pending-broadcast entry (fingerprinted CREATE op), look
|
|
2385
|
+
* for a matching chain order. If found: adopt it (set the opContext's
|
|
2386
|
+
* chainOrderId and continue with the existing planned slot).
|
|
2387
|
+
* 3. For pending entries with no chain match: virtualize (mark the
|
|
2388
|
+
* opContext as discarded; the planned slot stays empty until the
|
|
2389
|
+
* next planning cycle).
|
|
2390
|
+
* 4. Persist + log a structured [COW][UNCERTAIN] summary.
|
|
2391
|
+
*
|
|
2392
|
+
* After this method returns, the bot has either adopted the on-chain
|
|
2393
|
+
* result (good case — chain accepted but we didn't see the reply) or
|
|
2394
|
+
* accepted the discard (chain rejected or never received the op).
|
|
2395
|
+
*
|
|
2396
|
+
* @param {BroadcastUncertainError} err - The thrown error
|
|
2397
|
+
* @param {Array<Object>} opContexts - Original opContexts from the failed batch
|
|
2398
|
+
* @returns {Promise<Object>} Result object compatible with batch return shape
|
|
2399
|
+
*/
|
|
2400
|
+
async _reconcileAfterUncertainBroadcast(err, opContexts, options = {}) {
|
|
2401
|
+
if (options.fillLockAlreadyHeld !== true &&
|
|
2402
|
+
this.manager?._fillProcessingLock &&
|
|
2403
|
+
typeof this.manager._fillProcessingLock.acquire === 'function') {
|
|
2404
|
+
return this.manager._fillProcessingLock.acquire(async () => (this._reconcileAfterUncertainBroadcast(err, opContexts, {
|
|
2405
|
+
...options,
|
|
2406
|
+
fillLockAlreadyHeld: true
|
|
2407
|
+
})));
|
|
2408
|
+
}
|
|
2409
|
+
return this._reconcileAfterUncertainBroadcastImpl(err, opContexts, options);
|
|
2410
|
+
}
|
|
2411
|
+
async _reconcileAfterUncertainBroadcastImpl(err, opContexts, options) {
|
|
2412
|
+
const startedAt = Date.now();
|
|
2413
|
+
const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
|
|
2414
|
+
? Array.from(this.manager._pendingBroadcasts.values())
|
|
2415
|
+
: [];
|
|
2416
|
+
const createContextCount = opContexts.filter(c => c && c.kind === 'create').length;
|
|
2417
|
+
const nonCreateContextCount = opContexts.length - createContextCount;
|
|
2418
|
+
this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
|
|
2419
|
+
`creates=${createContextCount} nonCreates=${nonCreateContextCount} ` +
|
|
2420
|
+
`staleSinceMs=${err?.timeoutMs || 'n/a'}. Entering reconcile-then-decide.`, 'warn');
|
|
2421
|
+
if (!chainOrders?.readOpenOrders) {
|
|
2422
|
+
this.manager.logger.log('[COW][UNCERTAIN] readOpenOrders unavailable; falling back to structural resync only.', 'error');
|
|
2423
|
+
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
2424
|
+
await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders unavailable', { batchId: err?.batchId || null });
|
|
2425
|
+
}
|
|
2426
|
+
this._clearPendingBroadcasts();
|
|
2427
|
+
return { executed: false, hadRotation: false, uncertain: true };
|
|
2428
|
+
}
|
|
2429
|
+
// 1. Read the chain
|
|
2430
|
+
const accountRef = this.accountId || this.account?.id || this.account;
|
|
2431
|
+
let chainSnapshot = [];
|
|
2432
|
+
try {
|
|
2433
|
+
chainSnapshot = await chainOrders.readOpenOrders(accountRef);
|
|
2434
|
+
}
|
|
2435
|
+
catch (readErr) {
|
|
2436
|
+
this.manager.logger.log(`[COW][UNCERTAIN] readOpenOrders failed: ${readErr?.message || readErr}. ` +
|
|
2437
|
+
`Falling back to structural resync.`, 'error');
|
|
2438
|
+
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
2439
|
+
await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders failed', { batchId: err?.batchId || null, error: readErr?.message || String(readErr) });
|
|
2440
|
+
}
|
|
2441
|
+
this._clearPendingBroadcasts();
|
|
2442
|
+
return { executed: false, hadRotation: false, uncertain: true };
|
|
2443
|
+
}
|
|
2444
|
+
const adopted = [];
|
|
2445
|
+
const discarded = [];
|
|
2446
|
+
// 2. For each pending broadcast, look for a chain match.
|
|
2447
|
+
for (const entry of pending) {
|
|
2448
|
+
const match = this._findChainOrderForSlot(chainSnapshot, entry.slotId, {
|
|
2449
|
+
sell: entry.finalInts?.sell,
|
|
2450
|
+
receive: entry.finalInts?.receive,
|
|
2451
|
+
orderType: entry.orderType || entry.order?.type,
|
|
2452
|
+
fingerprint: entry.fingerprint
|
|
2453
|
+
});
|
|
2454
|
+
if (match) {
|
|
2455
|
+
adopted.push({ slotId: entry.slotId, chainOrderId: match.id });
|
|
2456
|
+
this.manager._pendingBroadcasts.delete(entry.fingerprint);
|
|
2457
|
+
}
|
|
2458
|
+
else {
|
|
2459
|
+
discarded.push({ slotId: entry.slotId });
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
// 3. Apply the result to the working grid.
|
|
2463
|
+
// - For adopted entries: re-run a structural sync so the manager picks up
|
|
2464
|
+
// the chain order into the planned slot. The pre-broadcast guard
|
|
2465
|
+
// already cleared the working grid, so we use a fresh sync.
|
|
2466
|
+
// - For discarded entries: leave the planned slot empty; the next
|
|
2467
|
+
// planning cycle will refill it.
|
|
2468
|
+
// - Short-circuit the re-sync when every CREATE in the batch was
|
|
2469
|
+
// fingerprinted AND adopted (the happy path of a slow but successful
|
|
2470
|
+
// chain). The chain state is already known for those slots, so a full
|
|
2471
|
+
// sync would just produce a burst of false-positive "no adoptable
|
|
2472
|
+
// slot" warnings for any non-CREATE orders that were already in
|
|
2473
|
+
// place before this batch.
|
|
2474
|
+
let hadRotation = false;
|
|
2475
|
+
const allCreatesAdopted = pending.length > 0
|
|
2476
|
+
&& pending.every(p => adopted.some(a => a.slotId === p.slotId));
|
|
2477
|
+
const shouldRunHeavySync = !(allCreatesAdopted && discarded.length === 0);
|
|
2478
|
+
if (shouldRunHeavySync) {
|
|
2479
|
+
try {
|
|
2480
|
+
if (chainSnapshot && chainSnapshot.length > 0 && this.manager?.syncFromOpenOrders) {
|
|
2481
|
+
await this.manager.syncFromOpenOrders(chainSnapshot, {
|
|
2482
|
+
skipAccounting: true,
|
|
2483
|
+
fillLockAlreadyHeld: true
|
|
2484
|
+
});
|
|
2485
|
+
hadRotation = true;
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
catch (syncErr) {
|
|
2489
|
+
this.manager.logger.log(`[COW][UNCERTAIN] syncFromOpenOrders failed during recovery: ${syncErr?.message || syncErr}`, 'error');
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
else {
|
|
2493
|
+
hadRotation = true;
|
|
2494
|
+
this.manager.logger.log(`[COW][UNCERTAIN] All ${adopted.length} fingerprinted CREATE(s) adopted; ` +
|
|
2495
|
+
`skipping heavy re-sync to avoid false-positive unmatched warnings.`, 'debug');
|
|
2496
|
+
}
|
|
2497
|
+
// 4. Log structured summary.
|
|
2498
|
+
const elapsedMs = Date.now() - startedAt;
|
|
2499
|
+
const heartbeatAgeMs = this._lastBroadcastHeartbeatAt
|
|
2500
|
+
? Date.now() - this._lastBroadcastHeartbeatAt
|
|
2501
|
+
: null;
|
|
2502
|
+
this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
|
|
2503
|
+
`staleSinceMs=${err?.timeoutMs || 'n/a'} heartbeatAgeMs=${heartbeatAgeMs ?? 'n/a'} ` +
|
|
2504
|
+
`adopted=${adopted.length} discarded=${discarded.length} elapsedMs=${elapsedMs}`, adopted.length > 0 ? 'info' : 'warn');
|
|
2505
|
+
if (adopted.length > 0) {
|
|
2506
|
+
this.manager.logger.log(`[COW][UNCERTAIN] Adopted chain orders: ${adopted
|
|
2507
|
+
.map(a => `${a.slotId}->${a.chainOrderId}`)
|
|
2508
|
+
.join(', ')}`, 'info');
|
|
2509
|
+
}
|
|
2510
|
+
if (discarded.length > 0) {
|
|
2511
|
+
this.manager.logger.log(`[COW][UNCERTAIN] Discarded planned CREATEs (no chain match): ${discarded
|
|
2512
|
+
.map(d => d.slotId)
|
|
2513
|
+
.join(', ')}`, 'warn');
|
|
2514
|
+
}
|
|
2515
|
+
this._clearPendingBroadcasts();
|
|
2516
|
+
// Post-recovery safety net: if there are still unmatched chain
|
|
2517
|
+
// orders after reconcile-then-decide, cancel ONE per cycle. The cap
|
|
2518
|
+
// is enforced inside the helper via _autoCancelOrphanCycleMarker.
|
|
2519
|
+
try {
|
|
2520
|
+
const autoCancelResult = await this._autoCancelOneUnmatchedOrphan();
|
|
2521
|
+
if (autoCancelResult.cancelled) {
|
|
2522
|
+
this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancelled orphan ${autoCancelResult.orderId} ` +
|
|
2523
|
+
`after recovery (cap=1/cycle).`, 'info');
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
catch (orphanErr) {
|
|
2527
|
+
this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancel pass failed: ${orphanErr?.message || orphanErr}`, 'warn');
|
|
2528
|
+
}
|
|
2529
|
+
// Persist master grid mutations from the reconciliation sync and any
|
|
2530
|
+
// orphan auto-cancellation that ran above. These apply outside the COW
|
|
2531
|
+
// broadcast path and would otherwise be in-memory only.
|
|
2532
|
+
if (typeof this.manager?.persistGrid === 'function') {
|
|
2533
|
+
await this.manager.persistGrid();
|
|
2534
|
+
}
|
|
2535
|
+
return { executed: false, hadRotation, uncertain: true, adopted, discarded };
|
|
2536
|
+
}
|
|
2537
|
+
/**
|
|
2538
|
+
* Auto-cancel a single unmatched chain order from the recovery snapshot.
|
|
2539
|
+
*
|
|
2540
|
+
* This is the post-recovery safety net: if, after
|
|
2541
|
+
* _reconcileAfterUncertainBroadcast runs, there are still chain orders
|
|
2542
|
+
* the bot doesn't recognize (e.g. from a network partition, or from a
|
|
2543
|
+
* daemon timeout that we couldn't even fingerprint), we cancel ONE of
|
|
2544
|
+
* them per call. Per-cycle cap = 1 — the next cycle will pick up the
|
|
2545
|
+
* next unmatched order if more remain.
|
|
2546
|
+
*
|
|
2547
|
+
* Safety conditions (ALL must hold):
|
|
2548
|
+
* 1. _pendingBroadcasts is empty (no in-flight recovery)
|
|
2549
|
+
* 2. _lastUnmatchedChainOrders is non-empty
|
|
2550
|
+
* 3. The current cycle has not already auto-cancelled an orphan
|
|
2551
|
+
* (tracked via this._autoCancelOrphanCycleMarker)
|
|
2552
|
+
*
|
|
2553
|
+
* Records the cancel via _recordOwnCancelOps so the fill consumer
|
|
2554
|
+
* doesn't trip the self-cancel guard.
|
|
2555
|
+
*
|
|
2556
|
+
* @returns {Promise<{cancelled: boolean, orderId?: string, reason?: string}>}
|
|
2557
|
+
*/
|
|
2558
|
+
async _autoCancelOneUnmatchedOrphan() {
|
|
2559
|
+
const cycleId = this._currentCycleId || 0;
|
|
2560
|
+
if (this._autoCancelOrphanCycleMarker === cycleId) {
|
|
2561
|
+
return { cancelled: false, reason: 'cap-reached-this-cycle' };
|
|
2562
|
+
}
|
|
2563
|
+
const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
|
|
2564
|
+
? this.manager._pendingBroadcasts.size
|
|
2565
|
+
: 0;
|
|
2566
|
+
if (pending > 0) {
|
|
2567
|
+
return { cancelled: false, reason: 'pending-broadcasts-active' };
|
|
2568
|
+
}
|
|
2569
|
+
const unmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
|
|
2570
|
+
? this.manager._lastUnmatchedChainOrders
|
|
2571
|
+
: [];
|
|
2572
|
+
if (unmatched.length === 0) {
|
|
2573
|
+
return { cancelled: false, reason: 'no-unmatched' };
|
|
2574
|
+
}
|
|
2575
|
+
const priceDriftOrphan = unmatched.find(u => u && u.reason === 'price-drift-orphan');
|
|
2576
|
+
const target = priceDriftOrphan || unmatched[0];
|
|
2577
|
+
const orderId = target?.id || target?.orderId || target?.chainOrderId;
|
|
2578
|
+
if (!orderId) {
|
|
2579
|
+
return { cancelled: false, reason: 'no-orderId' };
|
|
2580
|
+
}
|
|
2581
|
+
if (target?.fingerprint) {
|
|
2582
|
+
// Fingerprinted unmatched orders came from a pending broadcast.
|
|
2583
|
+
// The recovery path is the right place to handle them, not here.
|
|
2584
|
+
return { cancelled: false, reason: 'fingerprinted-handle-via-recovery' };
|
|
2585
|
+
}
|
|
2586
|
+
if (!chainOrders?.cancelOrder) {
|
|
2587
|
+
return { cancelled: false, reason: 'cancelOrder-unavailable' };
|
|
2588
|
+
}
|
|
2589
|
+
try {
|
|
2590
|
+
this._autoCancelOrphanCycleMarker = cycleId;
|
|
2591
|
+
this.manager.logger.log(`[COW] Auto-cancelling 1/${unmatched.length} unmatched chain order ` +
|
|
2592
|
+
`(${this._formatUnmatchedChainOrderForLog(target)}) — per-cycle cap=1.`, 'warn');
|
|
2593
|
+
await chainOrders.cancelOrder(this.account, this.privateKey, orderId);
|
|
2594
|
+
if (typeof chainOrders.recordOwnCancel === 'function') {
|
|
2595
|
+
chainOrders.recordOwnCancel(orderId);
|
|
2596
|
+
}
|
|
2597
|
+
return { cancelled: true, orderId };
|
|
2598
|
+
}
|
|
2599
|
+
catch (err) {
|
|
2600
|
+
this.manager.logger.log(`[COW] Auto-cancel of unmatched chain order ${orderId} failed: ${err?.message || err}`, 'error');
|
|
2601
|
+
return { cancelled: false, reason: 'cancel-failed', error: err?.message || String(err) };
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
2604
|
+
// Pair mode applies only when create contexts include both BUY and SELL.
|
|
2605
|
+
// Single-side create batches intentionally remain a single executeBatch.
|
|
2606
|
+
/**
|
|
2607
|
+
* Check whether to execute creates in outside-in pair mode (mixed BUY/SELL operations).
|
|
2608
|
+
* @param {Array<Object>} opContexts - Operation contexts array
|
|
2609
|
+
* @returns {boolean} True if pair mode should be used
|
|
2610
|
+
*/
|
|
2611
|
+
_shouldExecuteCreatePairMode(opContexts) {
|
|
2612
|
+
if (!Array.isArray(opContexts) || opContexts.length < 2)
|
|
2613
|
+
return false;
|
|
2614
|
+
if (!opContexts.every(ctx => ctx?.kind === 'create' && ctx?.order))
|
|
2615
|
+
return false;
|
|
2616
|
+
let hasBuy = false;
|
|
2617
|
+
let hasSell = false;
|
|
2618
|
+
for (const ctx of opContexts) {
|
|
2619
|
+
if (ctx.order.type === ORDER_TYPES.BUY)
|
|
2620
|
+
hasBuy = true;
|
|
2621
|
+
if (ctx.order.type === ORDER_TYPES.SELL)
|
|
2622
|
+
hasSell = true;
|
|
2623
|
+
if (hasBuy && hasSell)
|
|
2624
|
+
return true;
|
|
2625
|
+
}
|
|
2626
|
+
return false;
|
|
2627
|
+
}
|
|
2628
|
+
/**
|
|
2629
|
+
* Execute operations with retry on BroadcastUncertainError.
|
|
2630
|
+
* The daemon already retries internally against a 25s deadline.
|
|
2631
|
+
* If all expire, a fresh bot-level attempt buys a new 25s window.
|
|
2632
|
+
*
|
|
2633
|
+
* Skips retry when partialOnChainState is true (pair-mode grouped
|
|
2634
|
+
* execution where earlier groups already committed). Re-broadcasting
|
|
2635
|
+
* the full operations array would duplicate those creates on chain.
|
|
2636
|
+
*/
|
|
2637
|
+
async _executeWithRetryOnUncertain(operations, opContexts) {
|
|
2638
|
+
const MAX_RETRIES = 1;
|
|
2639
|
+
for (let attempt = 1;; attempt++) {
|
|
2640
|
+
try {
|
|
2641
|
+
return await this._executeOperationsWithStrategy(operations, opContexts);
|
|
2642
|
+
}
|
|
2643
|
+
catch (err) {
|
|
2644
|
+
const isRetriable = err instanceof BroadcastUncertainError
|
|
2645
|
+
&& !err.partialOnChainState
|
|
2646
|
+
&& attempt <= MAX_RETRIES;
|
|
2647
|
+
if (isRetriable) {
|
|
2648
|
+
this.manager.logger.log(`[COW] Broadcast uncertain (attempt ${attempt}/${MAX_RETRIES + 1}), retrying...`, 'warn');
|
|
2649
|
+
await this._ensureCredentialDaemonWritable('COW batch retry');
|
|
2650
|
+
continue;
|
|
2651
|
+
}
|
|
2652
|
+
throw err;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
/**
|
|
2657
|
+
* Execute blockchain operations with appropriate strategy (single batch or pair mode).
|
|
2658
|
+
* @param {Array<import('./types').CreatedOperation>} operations - Array of operation objects
|
|
2659
|
+
* @param {Array<Object>} opContexts - Array of operation context metadata (1:1 with operations)
|
|
2660
|
+
* @returns {Promise<{result: Object, opContexts: Array}>} Execution result with contexts
|
|
2661
|
+
*/
|
|
2662
|
+
async _executeOperationsWithStrategy(operations, opContexts) {
|
|
2663
|
+
if (!this._shouldExecuteCreatePairMode(opContexts)) {
|
|
2664
|
+
const result = await chainOrders.executeBatch(this.account, this.privateKey, operations);
|
|
2665
|
+
return { result, opContexts };
|
|
2666
|
+
}
|
|
2667
|
+
const createEntries = [];
|
|
2668
|
+
for (let i = 0; i < operations.length; i++) {
|
|
2669
|
+
createEntries.push({
|
|
2670
|
+
operation: operations[i],
|
|
2671
|
+
context: opContexts[i],
|
|
2672
|
+
});
|
|
2673
|
+
}
|
|
2674
|
+
const groups = this._buildOutsideInPairGroupsForCreateEntries(createEntries);
|
|
2675
|
+
const mergedOperationResults = [];
|
|
2676
|
+
const mergedRawResults = [];
|
|
2677
|
+
const mergedContexts = [];
|
|
2678
|
+
// Grouped execution is fail-fast, but NOT atomic across groups.
|
|
2679
|
+
// Each group is a separate on-chain transaction; if a later group fails,
|
|
2680
|
+
// earlier groups may already be confirmed on-chain.
|
|
2681
|
+
for (let idx = 0; idx < groups.length; idx++) {
|
|
2682
|
+
const group = groups[idx];
|
|
2683
|
+
const groupOps = group.map(e => e.operation);
|
|
2684
|
+
const groupContexts = group.map(e => e.context);
|
|
2685
|
+
this.manager.logger.log(`[COW] Broadcasting create pair group ${idx + 1}/${groups.length} (${groupOps.length} op${groupOps.length > 1 ? 's' : ''}, outside->center)`, 'info');
|
|
2686
|
+
let groupResult;
|
|
2687
|
+
try {
|
|
2688
|
+
groupResult = await chainOrders.executeBatch(this.account, this.privateKey, groupOps);
|
|
2689
|
+
}
|
|
2690
|
+
catch (err) {
|
|
2691
|
+
const groupsBroadcast = idx;
|
|
2692
|
+
const groupsTotal = groups.length;
|
|
2693
|
+
const broadcastedOperationCount = mergedContexts.length;
|
|
2694
|
+
this.manager.logger.log(`[COW] Grouped create execution failed at group ${idx + 1}/${groupsTotal}; ${groupsBroadcast} group(s) already broadcast (${broadcastedOperationCount} op context(s)). Partial on-chain state is possible.`, 'error');
|
|
2695
|
+
err.partialOnChainState = groupsBroadcast > 0;
|
|
2696
|
+
err.groupsBroadcast = groupsBroadcast;
|
|
2697
|
+
err.groupsTotal = groupsTotal;
|
|
2698
|
+
err.broadcastedOperationCount = broadcastedOperationCount;
|
|
2699
|
+
throw err;
|
|
2700
|
+
}
|
|
2701
|
+
const groupOpResults = this._extractOperationResults(groupResult);
|
|
2702
|
+
mergedOperationResults.push(...groupOpResults);
|
|
2703
|
+
mergedRawResults.push(groupResult?.raw || null);
|
|
2704
|
+
mergedContexts.push(...groupContexts);
|
|
2705
|
+
}
|
|
2706
|
+
return {
|
|
2707
|
+
result: {
|
|
2708
|
+
success: true,
|
|
2709
|
+
raw: {
|
|
2710
|
+
grouped: true,
|
|
2711
|
+
groupsExecuted: groups.length,
|
|
2712
|
+
groupResults: mergedRawResults,
|
|
2713
|
+
},
|
|
2714
|
+
operation_results: mergedOperationResults,
|
|
2715
|
+
grouped: true,
|
|
2716
|
+
groupsExecuted: groups.length
|
|
2717
|
+
},
|
|
2718
|
+
opContexts: mergedContexts
|
|
2719
|
+
};
|
|
2720
|
+
}
|
|
2721
|
+
/**
|
|
2722
|
+
* Validate that operations can be executed with available funds before broadcasting.
|
|
2723
|
+
* Checks sufficient available funds for all operations.
|
|
2724
|
+
* @param {Array} operations - Operations to validate
|
|
2725
|
+
* @param {Object} assetA - Asset A metadata (id, precision, symbol)
|
|
2726
|
+
* @param {Object} assetB - Asset B metadata (id, precision, symbol)
|
|
2727
|
+
* @returns {Object} { isValid: boolean, summary: string }
|
|
2728
|
+
* @private
|
|
2729
|
+
*/
|
|
2730
|
+
_validateOperationFunds(operations, assetA, assetB) {
|
|
2731
|
+
if (!operations || operations.length === 0) {
|
|
2732
|
+
return { isValid: true, summary: 'No operations to validate' };
|
|
2733
|
+
}
|
|
2734
|
+
const { blockchainToFloat, floatToBlockchainInt, quantizeFloat } = require('./order/utils/math');
|
|
2735
|
+
const snap = this.manager.getChainFundsSnapshot();
|
|
2736
|
+
const netRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
|
|
2737
|
+
const runningRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
|
|
2738
|
+
const peakRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
|
|
2739
|
+
// Sum amounts and check individual order sizes
|
|
2740
|
+
for (const op of operations) {
|
|
2741
|
+
if (!op?.op_data)
|
|
2742
|
+
continue;
|
|
2743
|
+
let sellAssetId = null;
|
|
2744
|
+
let sellAmountInt = 0;
|
|
2745
|
+
if (op.op_name === 'limit_order_create') {
|
|
2746
|
+
sellAssetId = op.op_data.amount_to_sell?.asset_id;
|
|
2747
|
+
sellAmountInt = op.op_data.amount_to_sell?.amount;
|
|
2748
|
+
}
|
|
2749
|
+
else if (op.op_name === 'limit_order_update') {
|
|
2750
|
+
// In limit_order_update, new_price.base is the amount to sell
|
|
2751
|
+
sellAssetId = op.op_data.new_price?.base?.asset_id;
|
|
2752
|
+
sellAmountInt = op.op_data.new_price?.base?.amount;
|
|
2753
|
+
}
|
|
2754
|
+
if (sellAssetId && (sellAmountInt !== undefined && sellAmountInt !== null)) {
|
|
2755
|
+
const precision = (sellAssetId === assetA.id) ? assetA.precision : assetB.precision;
|
|
2756
|
+
const assetSymbol = (sellAssetId === assetA.id) ? assetA.symbol : assetB.symbol;
|
|
2757
|
+
// CRITICAL SAFETY CHECK: Ensure amount is greater than zero
|
|
2758
|
+
if (Number(sellAmountInt) <= 0) {
|
|
2759
|
+
return {
|
|
2760
|
+
isValid: false,
|
|
2761
|
+
summary: `[VALIDATION] CRITICAL: Zero amount order detected for ${assetSymbol} (assetId=${sellAssetId})`,
|
|
2762
|
+
violations: [{ asset: assetSymbol, sizeInt: sellAmountInt, reason: 'Zero amount' }]
|
|
2763
|
+
};
|
|
2764
|
+
}
|
|
2765
|
+
// Track signed per-op deltas in operation order.
|
|
2766
|
+
// CREATE consumes full amount; UPDATE consumes/releases delta.
|
|
2767
|
+
let signedDelta = 0;
|
|
2768
|
+
if (op.op_name === 'limit_order_update') {
|
|
2769
|
+
const deltaAssetId = op.op_data.delta_amount_to_sell?.asset_id;
|
|
2770
|
+
const deltaSellInt = op.op_data.delta_amount_to_sell?.amount;
|
|
2771
|
+
if (deltaAssetId === sellAssetId && Number.isFinite(Number(deltaSellInt))) {
|
|
2772
|
+
signedDelta = blockchainToFloat(deltaSellInt, precision);
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2775
|
+
else {
|
|
2776
|
+
signedDelta = blockchainToFloat(sellAmountInt, precision);
|
|
2777
|
+
}
|
|
2778
|
+
netRequiredFunds[sellAssetId] = quantizeFloat((netRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
|
|
2779
|
+
runningRequiredFunds[sellAssetId] = quantizeFloat((runningRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
|
|
2780
|
+
const nextPeak = Math.max(Number(peakRequiredFunds[sellAssetId] || 0), Number(runningRequiredFunds[sellAssetId] || 0));
|
|
2781
|
+
peakRequiredFunds[sellAssetId] = quantizeFloat(nextPeak, precision);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
// Calculate available funds - CRITICAL FIX: Check against FREE balance, not free+required
|
|
2785
|
+
// Bug: Previous logic added requiredFunds to available, making validation meaningless
|
|
2786
|
+
// Correct logic: available = chainFree (current free balance)
|
|
2787
|
+
// If required > available, batch will fail on execution
|
|
2788
|
+
const availableFunds = {
|
|
2789
|
+
[assetA.id]: quantizeFloat(snap.chainFreeSell || 0, assetA.precision),
|
|
2790
|
+
[assetB.id]: quantizeFloat(snap.chainFreeBuy || 0, assetB.precision)
|
|
2791
|
+
};
|
|
2792
|
+
// Check for fund violations using quantized comparison
|
|
2793
|
+
const fundViolations = [];
|
|
2794
|
+
for (const assetId in peakRequiredFunds) {
|
|
2795
|
+
const required = peakRequiredFunds[assetId];
|
|
2796
|
+
const netRequired = netRequiredFunds[assetId] || 0;
|
|
2797
|
+
const available = availableFunds[assetId] || 0;
|
|
2798
|
+
// Use precision-aware comparison
|
|
2799
|
+
const prec = (assetId === assetA.id) ? assetA.precision : assetB.precision;
|
|
2800
|
+
if (floatToBlockchainInt(required, prec) > floatToBlockchainInt(available, prec)) {
|
|
2801
|
+
fundViolations.push({
|
|
2802
|
+
asset: assetId === assetA.id ? assetA.symbol : assetB.symbol,
|
|
2803
|
+
required,
|
|
2804
|
+
netRequired,
|
|
2805
|
+
available,
|
|
2806
|
+
deficit: quantizeFloat(required - available, prec)
|
|
2807
|
+
});
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2810
|
+
if (fundViolations.length > 0) {
|
|
2811
|
+
let summary = `[VALIDATION] Fund validation FAILED:\n`;
|
|
2812
|
+
for (const v of fundViolations) {
|
|
2813
|
+
summary += ` ${v.asset}: peakRequired=${Format.formatAmount8(v.required)}, netRequired=${Format.formatAmount8(v.netRequired)}, available=${Format.formatAmount8(v.available)}, deficit=${Format.formatAmount8(v.deficit)}\n`;
|
|
2814
|
+
}
|
|
2815
|
+
return { isValid: false, summary: summary.trim(), violations: fundViolations };
|
|
2816
|
+
}
|
|
2817
|
+
const summary = `[VALIDATION] PASSED: ${operations.length} operations`;
|
|
2818
|
+
return { isValid: true, summary };
|
|
2819
|
+
}
|
|
2820
|
+
/**
|
|
2821
|
+
* Resolve the ideal size from an order-like object with fallback.
|
|
2822
|
+
* @param {Object|null} orderLike - Order-like object with optional idealSize/size nested properties
|
|
2823
|
+
* @param {number|null} [fallbackSize=null] - Fallback size if none found
|
|
2824
|
+
* @returns {number|null} Resolved size or null
|
|
2825
|
+
*/
|
|
2826
|
+
_resolveIdealSizeForValidation(orderLike, fallbackSize = null) {
|
|
2827
|
+
const candidates = [
|
|
2828
|
+
orderLike?.idealSize,
|
|
2829
|
+
orderLike?.order?.idealSize,
|
|
2830
|
+
orderLike?.size,
|
|
2831
|
+
orderLike?.order?.size,
|
|
2832
|
+
fallbackSize
|
|
2833
|
+
];
|
|
2834
|
+
for (const candidate of candidates) {
|
|
2835
|
+
const numeric = Number(candidate);
|
|
2836
|
+
if (Number.isFinite(numeric) && numeric > 0) {
|
|
2837
|
+
return numeric;
|
|
2838
|
+
}
|
|
2839
|
+
}
|
|
2840
|
+
return null;
|
|
2841
|
+
}
|
|
2842
|
+
/**
|
|
2843
|
+
* Validate that an order size is safe to execute (above minimum dust thresholds).
|
|
2844
|
+
* @param {number} size - Order size to validate
|
|
2845
|
+
* @param {string} type - ORDER_TYPES.BUY or ORDER_TYPES.SELL
|
|
2846
|
+
* @param {Object|null} [orderLike=null] - Optional order-like object for ideal size comparison
|
|
2847
|
+
* @param {number|null} [fallbackSize=null] - Fallback ideal size
|
|
2848
|
+
* @returns {import('./types').OrderValidationResult}
|
|
2849
|
+
*/
|
|
2850
|
+
_validateOrderSizeForExecution(size, type, orderLike = null, fallbackSize = null) {
|
|
2851
|
+
return validateOrderSize(size, type, this.manager.assets, this.config.gridLimits?.MIN_ORDER_SIZE_FACTOR, this._resolveIdealSizeForValidation(orderLike, fallbackSize), this.config.gridLimits?.PARTIAL_DUST_THRESHOLD_PERCENTAGE);
|
|
2852
|
+
}
|
|
2853
|
+
/**
|
|
2854
|
+
* Execute a batch of order operations if the rebalance result has executable actions.
|
|
2855
|
+
* @param {Object} rebalanceResult - COW rebalance result with actions
|
|
2856
|
+
* @param {string} [contextLabel='rebalance'] - Context label for logging
|
|
2857
|
+
* @returns {Promise<Object>} Batch execution result
|
|
2858
|
+
*/
|
|
2859
|
+
async _executeBatchIfNeeded(rebalanceResult, contextLabel = 'rebalance') {
|
|
2860
|
+
if (!hasExecutableActions(rebalanceResult)) {
|
|
2861
|
+
this.manager?.logger?.log?.(`[COW] No actions needed for ${contextLabel}`, 'debug');
|
|
2862
|
+
// Clear REBALANCING state even when there are no actions to execute.
|
|
2863
|
+
// _applySafeRebalanceCOW sets REBALANCING before calling the COW engine;
|
|
2864
|
+
// if the engine returns an empty actions list (not aborted), the state
|
|
2865
|
+
// would otherwise remain stuck at REBALANCING permanently, blocking
|
|
2866
|
+
// all subsequent fill processing and rebalance attempts.
|
|
2867
|
+
this.manager?._clearWorkingGridRef?.();
|
|
2868
|
+
// Persist master grid mutations that may have occurred outside COW
|
|
2869
|
+
// broadcast (e.g., partial-fill size updates applied directly by the
|
|
2870
|
+
// sync engine). Without this, a partial fill that does not trigger a
|
|
2871
|
+
// COW rebalance leaves the in-memory master grid ahead of the on-disk
|
|
2872
|
+
// snapshot, so the updated size is lost on restart.
|
|
2873
|
+
if (typeof this.manager?.persistGrid === 'function') {
|
|
2874
|
+
const persistResult = await this.manager.persistGrid();
|
|
2875
|
+
// persistGrid returns:
|
|
2876
|
+
// - { isValid: true, skipped: true, suspended: true } when persistence is suspended
|
|
2877
|
+
// - { isValid: false, reason } when validation rejects the state
|
|
2878
|
+
// - { isValid: true } on success (skipped is absent/undefined)
|
|
2879
|
+
// Warn only when validation actively rejected the state — not
|
|
2880
|
+
// when persistence was suspended (handled by the persistence
|
|
2881
|
+
// gate elsewhere).
|
|
2882
|
+
if (persistResult
|
|
2883
|
+
&& persistResult.skipped !== true
|
|
2884
|
+
&& persistResult.isValid === false) {
|
|
2885
|
+
this.manager.logger.log(`[COW] Master grid persistence validation failed after no-action batch (${contextLabel}): ${persistResult.reason || 'unknown'}`, 'warn');
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
2888
|
+
return { executed: false, hadRotation: false, skippedNoActions: true };
|
|
2889
|
+
}
|
|
2890
|
+
return await this.updateOrdersOnChainBatch(rebalanceResult);
|
|
2891
|
+
}
|
|
2892
|
+
/**
|
|
2893
|
+
* Process filled orders in capped batches per FILL_PROCESSING.MAX_FILL_BATCH_SIZE.
|
|
2894
|
+
* Each chunk triggers its own processFilledOrders → COW plan → broadcast cycle.
|
|
2895
|
+
*
|
|
2896
|
+
* @param {Array} fills - Filled order objects to process
|
|
2897
|
+
* @param {Set|null} excl - Exclusion set (order IDs to skip)
|
|
2898
|
+
* @param {string} contextLabel - Label for logging and batch context
|
|
2899
|
+
* @param {Object} [options={}] - Passed through to processFilledOrders
|
|
2900
|
+
* @returns {{aborted: boolean}}
|
|
2901
|
+
*/
|
|
2902
|
+
async _processFillsWithBatching(fills, excl, contextLabel, options = {}) {
|
|
2903
|
+
if (!fills || fills.length === 0) {
|
|
2904
|
+
return { aborted: false };
|
|
2905
|
+
}
|
|
2906
|
+
const managerLog = this.manager?.logger?.log?.bind(this.manager.logger) || (() => { });
|
|
2907
|
+
const maxBatch = this._getMaxFillBatchSize();
|
|
2908
|
+
const totalFills = fills.length;
|
|
2909
|
+
const useUnifiedPlan = totalFills <= maxBatch;
|
|
2910
|
+
const modeLabel = useUnifiedPlan ? 'unified' : 'chunked';
|
|
2911
|
+
managerLog(`Processing ${totalFills} filled orders (${modeLabel}, baseBatch=${useUnifiedPlan ? totalFills : maxBatch})...`, 'info');
|
|
2912
|
+
if (typeof this.manager?.pauseFundRecalc === 'function') {
|
|
2913
|
+
this.manager.pauseFundRecalc();
|
|
2914
|
+
}
|
|
2915
|
+
try {
|
|
2916
|
+
let i = 0;
|
|
2917
|
+
while (i < totalFills) {
|
|
2918
|
+
const remaining = totalFills - i;
|
|
2919
|
+
const currentBatchSize = useUnifiedPlan ? remaining : Math.min(maxBatch, remaining);
|
|
2920
|
+
const batchEnd = Math.min(i + currentBatchSize, totalFills);
|
|
2921
|
+
const fillBatch = fills.slice(i, batchEnd);
|
|
2922
|
+
i = batchEnd;
|
|
2923
|
+
const batchIds = fillBatch.map(f => f.id).join(', ');
|
|
2924
|
+
const label = `${contextLabel} [${batchIds}]`;
|
|
2925
|
+
managerLog(`>>> Processing fill set ${label} (${i}/${totalFills})`, 'info');
|
|
2926
|
+
let fullExcludeSet = excl || new Set();
|
|
2927
|
+
if (!useUnifiedPlan) {
|
|
2928
|
+
const batchIdSet = new Set(fillBatch.map(f => f.id));
|
|
2929
|
+
fullExcludeSet = new Set(excl || []);
|
|
2930
|
+
for (const other of fills) {
|
|
2931
|
+
if (batchIdSet.has(other.id))
|
|
2932
|
+
continue;
|
|
2933
|
+
if (other.orderId)
|
|
2934
|
+
fullExcludeSet.add(other.orderId);
|
|
2935
|
+
if (other.id)
|
|
2936
|
+
fullExcludeSet.add(other.id);
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
const rebalanceResult = await this.manager.processFilledOrders(fillBatch, fullExcludeSet, options);
|
|
2940
|
+
const batchResult = await this._executeBatchIfNeeded(rebalanceResult, label);
|
|
2941
|
+
if (batchResult?.abortedForIllegalState || batchResult?.abortedForAccountingFailure) {
|
|
2942
|
+
managerLog(`[HARD-ABORT] ${label} aborted due to critical state. Skipping remaining fills.`, 'error');
|
|
2943
|
+
return { aborted: true };
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2946
|
+
}
|
|
2947
|
+
finally {
|
|
2948
|
+
if (typeof this.manager?.resumeFundRecalc === 'function') {
|
|
2949
|
+
await this.manager.resumeFundRecalc();
|
|
2950
|
+
}
|
|
2951
|
+
// End-of-tick safety net: any direct master-grid mutations that
|
|
2952
|
+
// did not reach a known persistGrid() site are still persisted
|
|
2953
|
+
// here. This catches partial-only fill batches that update slot
|
|
2954
|
+
// sizes in-memory via _applyOrderUpdate without triggering a
|
|
2955
|
+
// COW rebalance (the bug that left slot-108 with stale size
|
|
2956
|
+
// 0.3293 instead of 0.0001 across restarts).
|
|
2957
|
+
if (typeof this.manager?.flushGridDirty === 'function') {
|
|
2958
|
+
await this.manager.flushGridDirty('end-of-tick fill processing');
|
|
2959
|
+
}
|
|
2960
|
+
}
|
|
2961
|
+
return { aborted: false };
|
|
2962
|
+
}
|
|
2963
|
+
/**
|
|
2964
|
+
* Check if the bot requires credential daemon for write operations.
|
|
2965
|
+
* @returns {boolean}
|
|
2966
|
+
*/
|
|
2967
|
+
_isCredentialDaemonWriteRequired() {
|
|
2968
|
+
return getKeyStore().isDaemonSigningKey(this.privateKey);
|
|
2969
|
+
}
|
|
2970
|
+
/**
|
|
2971
|
+
* Suspend grid persistence due to credential daemon outage.
|
|
2972
|
+
* @param {string} reason - Reason for suspension
|
|
2973
|
+
* @returns {void}
|
|
2974
|
+
*/
|
|
2975
|
+
_suspendGridPersistenceForCredentialOutage(reason) {
|
|
2976
|
+
if (typeof this.manager?.suspendGridPersistence === 'function') {
|
|
2977
|
+
this.manager.suspendGridPersistence(reason);
|
|
2978
|
+
}
|
|
2979
|
+
}
|
|
2980
|
+
/**
|
|
2981
|
+
* Resume grid persistence after credential daemon recovery.
|
|
2982
|
+
* @param {string} reason - Reason for resuming
|
|
2983
|
+
* @returns {void}
|
|
2984
|
+
*/
|
|
2985
|
+
_resumeGridPersistenceAfterCredentialRecovery(reason) {
|
|
2986
|
+
if (typeof this.manager?.resumeGridPersistence === 'function') {
|
|
2987
|
+
this.manager.resumeGridPersistence(reason);
|
|
2988
|
+
}
|
|
2989
|
+
}
|
|
2990
|
+
/**
|
|
2991
|
+
* Ensure the credential daemon is writable before broadcasting operations.
|
|
2992
|
+
* @param {string} [contextLabel='write batch'] - Context label for logging
|
|
2993
|
+
* @returns {Promise<void>}
|
|
2994
|
+
* @throws {Error} With code CREDENTIAL_DAEMON_UNAVAILABLE if daemon is down
|
|
2995
|
+
*/
|
|
2996
|
+
async _ensureCredentialDaemonWritable(contextLabel = 'write batch') {
|
|
2997
|
+
if (!this._isCredentialDaemonWriteRequired()) {
|
|
2998
|
+
return;
|
|
2999
|
+
}
|
|
3000
|
+
try {
|
|
3001
|
+
if (this.privateKey && getKeyStore().isDaemonSigningKey(this.privateKey)) {
|
|
3002
|
+
await chainKeys.pingDaemon(this.privateKey.accountName, Math.min(TIMING.DAEMON_PING_TIMEOUT_MS, TIMING.DAEMON_STARTUP_TIMEOUT_MS), { socketPath: this.privateKey.socketPath });
|
|
3003
|
+
}
|
|
3004
|
+
}
|
|
3005
|
+
catch (err) {
|
|
3006
|
+
const message = `Credential daemon unavailable before ${contextLabel}: ${err.message}`;
|
|
3007
|
+
this._credentialDaemonDown = true;
|
|
3008
|
+
this._credentialRecoveryNeeded = true;
|
|
3009
|
+
this._suspendGridPersistenceForCredentialOutage(message);
|
|
3010
|
+
this.manager?.logger?.log?.(`[CREDENTIAL] ${message}. Write operations paused; re-unlock with node pm2.`, 'error');
|
|
3011
|
+
const wrapped = new Error(message);
|
|
3012
|
+
wrapped.code = DAEMON_CODES.CREDENTIAL_DAEMON_UNAVAILABLE;
|
|
3013
|
+
wrapped.cause = err;
|
|
3014
|
+
throw wrapped;
|
|
3015
|
+
}
|
|
3016
|
+
}
|
|
3017
|
+
/**
|
|
3018
|
+
* Check if an error is related to credential daemon unavailability.
|
|
3019
|
+
* @param {Error|*} err - Error to check
|
|
3020
|
+
* @returns {boolean}
|
|
3021
|
+
*/
|
|
3022
|
+
_isCredentialDaemonError(err) {
|
|
3023
|
+
if (!err)
|
|
3024
|
+
return false;
|
|
3025
|
+
if (err.code === DAEMON_CODES.CREDENTIAL_DAEMON_UNAVAILABLE)
|
|
3026
|
+
return true;
|
|
3027
|
+
const message = String(err.message || '');
|
|
3028
|
+
return /Credential daemon|Daemon connection failed|daemon .*unavailable|dexbot-cred-daemon\.sock|ECONNREFUSED|ENOENT/.test(message);
|
|
3029
|
+
}
|
|
3030
|
+
/**
|
|
3031
|
+
* Run state recovery after credential daemon is restored.
|
|
3032
|
+
* @returns {Promise<void>}
|
|
3033
|
+
*/
|
|
3034
|
+
async _runCredentialRecoveryAfterDaemonRestored() {
|
|
3035
|
+
if (this._credentialRecoveryInFlight || !this._credentialRecoveryNeeded || this._shuttingDown) {
|
|
3036
|
+
return;
|
|
3037
|
+
}
|
|
3038
|
+
if (this.manager?._state?.isBootstrapping?.() || this.manager?._state?.isBroadcastingActive?.()) {
|
|
3039
|
+
if (!this._credentialRecoveryDeferredTimer) {
|
|
3040
|
+
this.manager?.logger?.log?.('[CREDENTIAL] Deferring credential recovery until startup/broadcast activity is idle.', 'info');
|
|
3041
|
+
this._credentialRecoveryDeferredTimer = setTimeout(() => {
|
|
3042
|
+
this._credentialRecoveryDeferredTimer = null;
|
|
3043
|
+
this._runCredentialRecoveryAfterDaemonRestored().catch(err => {
|
|
3044
|
+
this.manager?.logger?.log?.(`[CREDENTIAL] Deferred recovery failed: ${err.message}`, 'error');
|
|
3045
|
+
if (this.manager) {
|
|
3046
|
+
this.manager._recoveryState = this.manager._recoveryState || {};
|
|
3047
|
+
this.manager._recoveryState.lastFailureAt = Date.now();
|
|
3048
|
+
}
|
|
3049
|
+
});
|
|
3050
|
+
}, 1000);
|
|
3051
|
+
}
|
|
3052
|
+
return;
|
|
3053
|
+
}
|
|
3054
|
+
this._credentialRecoveryInFlight = true;
|
|
3055
|
+
try {
|
|
3056
|
+
this.manager?.logger?.log?.('[CREDENTIAL] Credential daemon restored; reconciling chain state before resuming write batches.', 'info');
|
|
3057
|
+
this._resumeGridPersistenceAfterCredentialRecovery('credential recovery started');
|
|
3058
|
+
const runRecovery = async () => {
|
|
3059
|
+
await this._triggerStateRecoverySync('credential daemon restored');
|
|
3060
|
+
await this._runGridMaintenance('credential-recovery', { fillLockAlreadyHeld: true });
|
|
3061
|
+
};
|
|
3062
|
+
if (this.manager?._fillProcessingLock) {
|
|
3063
|
+
await this.manager._fillProcessingLock.acquire(runRecovery);
|
|
3064
|
+
}
|
|
3065
|
+
else {
|
|
3066
|
+
await runRecovery();
|
|
3067
|
+
}
|
|
3068
|
+
this._credentialRecoveryNeeded = false;
|
|
3069
|
+
this.manager?.logger?.log?.('[CREDENTIAL] Credential recovery sync complete.', 'info');
|
|
3070
|
+
}
|
|
3071
|
+
catch (err) {
|
|
3072
|
+
this._credentialRecoveryNeeded = true;
|
|
3073
|
+
this._suspendGridPersistenceForCredentialOutage(`credential recovery failed: ${err.message}`);
|
|
3074
|
+
this.manager?.logger?.log?.(`[CREDENTIAL] Credential recovery sync failed: ${err.message}. Writes remain guarded by preflight.`, 'error');
|
|
3075
|
+
}
|
|
3076
|
+
finally {
|
|
3077
|
+
this._credentialRecoveryInFlight = false;
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
/**
|
|
3081
|
+
* Start the credential daemon watchdog interval that periodically probes daemon health.
|
|
3082
|
+
* @returns {void}
|
|
3083
|
+
*/
|
|
3084
|
+
_setupCredentialDaemonWatchdogInterval() {
|
|
3085
|
+
if (this._credentialDaemonWatchdogInterval) {
|
|
3086
|
+
clearInterval(this._credentialDaemonWatchdogInterval);
|
|
3087
|
+
this._credentialDaemonWatchdogInterval = null;
|
|
3088
|
+
}
|
|
3089
|
+
if (!this._isCredentialDaemonWriteRequired()) {
|
|
3090
|
+
this._credentialDaemonDown = false;
|
|
3091
|
+
return;
|
|
3092
|
+
}
|
|
3093
|
+
const intervalMs = Math.max(TIMING.DAEMON_PING_TIMEOUT_MS, TIMING.CREDENTIAL_DAEMON_WATCHDOG_MS);
|
|
3094
|
+
const probe = async () => {
|
|
3095
|
+
if (this._shuttingDown || !this._isCredentialDaemonWriteRequired())
|
|
3096
|
+
return;
|
|
3097
|
+
// Guard against overlapping ticks: if the previous probe is still
|
|
3098
|
+
// running (slow chain / stall), skip rather than queue a second
|
|
3099
|
+
// pingDaemon and a second recovery attempt.
|
|
3100
|
+
if (this._credentialDaemonWatchdogInFlight)
|
|
3101
|
+
return;
|
|
3102
|
+
this._credentialDaemonWatchdogInFlight = true;
|
|
3103
|
+
try {
|
|
3104
|
+
const token = this.privateKey;
|
|
3105
|
+
try {
|
|
3106
|
+
if (getKeyStore().isDaemonSigningKey(token)) {
|
|
3107
|
+
await chainKeys.pingDaemon(token.accountName, 2000, { socketPath: token.socketPath });
|
|
3108
|
+
}
|
|
3109
|
+
if (this._credentialDaemonDown) {
|
|
3110
|
+
this.manager?.logger?.log?.('[CREDENTIAL] Credential daemon responsive again.', 'info');
|
|
3111
|
+
}
|
|
3112
|
+
this._credentialDaemonDown = false;
|
|
3113
|
+
await this._runCredentialRecoveryAfterDaemonRestored();
|
|
3114
|
+
}
|
|
3115
|
+
catch (err) {
|
|
3116
|
+
if (!this._credentialDaemonDown) {
|
|
3117
|
+
const errMsg = String(err.message || '');
|
|
3118
|
+
let hint = '';
|
|
3119
|
+
if (errMsg.includes('ENOENT')) {
|
|
3120
|
+
hint = `Socket file missing at ${token.socketPath}. The credential daemon process may have been killed (e.g. by stray Ctrl-C). Restart it with: node pm2 restart dexbot-cred. If the problem persists, check the daemon log: profiles/logs/dexbot-cred.log`;
|
|
3121
|
+
}
|
|
3122
|
+
else if (errMsg.includes('ECONNREFUSED')) {
|
|
3123
|
+
hint = `Connection refused at ${token.socketPath}. The daemon may be in a zombie state or restarting. Try: node pm2 restart dexbot-cred.`;
|
|
3124
|
+
}
|
|
3125
|
+
else if (errMsg.includes('timeout')) {
|
|
3126
|
+
hint = `Probe timed out. The daemon may be under heavy load or blocked. Check profiles/logs/dexbot-cred.log.`;
|
|
3127
|
+
}
|
|
3128
|
+
else {
|
|
3129
|
+
hint = `Write operations will remain paused until re-unlocked with node pm2.`;
|
|
3130
|
+
}
|
|
3131
|
+
this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog failed: ${err.message}. ${hint}`, 'error');
|
|
3132
|
+
}
|
|
3133
|
+
this._credentialDaemonDown = true;
|
|
3134
|
+
this._suspendGridPersistenceForCredentialOutage(`credential daemon watchdog failed: ${err.message}`);
|
|
3135
|
+
}
|
|
3136
|
+
}
|
|
3137
|
+
finally {
|
|
3138
|
+
this._credentialDaemonWatchdogInFlight = false;
|
|
3139
|
+
}
|
|
3140
|
+
};
|
|
3141
|
+
this._credentialDaemonWatchdogInterval = setInterval(() => {
|
|
3142
|
+
probe().catch(err => {
|
|
3143
|
+
this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog error: ${err.message}`, 'warn');
|
|
3144
|
+
});
|
|
3145
|
+
}, intervalMs);
|
|
3146
|
+
if (typeof this._credentialDaemonWatchdogInterval.unref === 'function') {
|
|
3147
|
+
this._credentialDaemonWatchdogInterval.unref();
|
|
3148
|
+
}
|
|
3149
|
+
void probe();
|
|
3150
|
+
this._log(`Credential daemon watchdog started (${Math.round(intervalMs / TIMING.MILLISECONDS_PER_SECOND)}s interval)`);
|
|
3151
|
+
}
|
|
3152
|
+
/**
|
|
3153
|
+
* Stop the credential daemon watchdog interval.
|
|
3154
|
+
* @returns {void}
|
|
3155
|
+
*/
|
|
3156
|
+
_stopCredentialDaemonWatchdogInterval() {
|
|
3157
|
+
if (this._credentialDaemonWatchdogInterval) {
|
|
3158
|
+
clearInterval(this._credentialDaemonWatchdogInterval);
|
|
3159
|
+
this._credentialDaemonWatchdogInterval = null;
|
|
3160
|
+
}
|
|
3161
|
+
}
|
|
3162
|
+
/**
|
|
3163
|
+
* Executes a batch of order operations on the blockchain using COW pattern.
|
|
3164
|
+
* Master grid is only updated after successful blockchain confirmation.
|
|
3165
|
+
* @param {Object} rebalanceResult - COW result containing workingGrid + actions.
|
|
3166
|
+
* @returns {Promise<Object>} The batch result.
|
|
3167
|
+
*/
|
|
3168
|
+
async updateOrdersOnChainBatch(rebalanceResult) {
|
|
3169
|
+
if (!rebalanceResult || !rebalanceResult.workingGrid) {
|
|
3170
|
+
const reason = 'NON_COW_PAYLOAD';
|
|
3171
|
+
this.manager?.logger?.log?.(`[COW] Rejected non-COW batch payload. Use updateOrdersOnChainPlan() for plan inputs.`, 'error');
|
|
3172
|
+
return { executed: false, aborted: true, reason };
|
|
3173
|
+
}
|
|
3174
|
+
return await this._updateOrdersOnChainBatchCOW(rebalanceResult);
|
|
3175
|
+
}
|
|
3176
|
+
/**
|
|
3177
|
+
* Converts simple plan payloads (place/update/rotate/cancel) into a COW batch.
|
|
3178
|
+
* Used by spread/divergence maintenance and bootstrap helpers.
|
|
3179
|
+
* @param {Object|Array} plan - Plan object or array of ordersToPlace
|
|
3180
|
+
* @returns {Promise<Object>} Batch execution result
|
|
3181
|
+
*/
|
|
3182
|
+
async updateOrdersOnChainPlan(plan) {
|
|
3183
|
+
const cowResult = this._buildCowResultFromPlan(plan);
|
|
3184
|
+
return await this._updateOrdersOnChainBatchCOW(cowResult);
|
|
3185
|
+
}
|
|
3186
|
+
/**
|
|
3187
|
+
* Build COW actions array from a simple plan object or array of ordersToPlace.
|
|
3188
|
+
* @param {Object|Array} plan - Plan object with ordersToPlace/ordersToRotate/ordersToUpdate/ordersToCancel, or array of ordersToPlace
|
|
3189
|
+
* @returns {Array<{type: string, id: string, order?: Object, orderId?: string, newSize?: number, newPrice?: number, newGridId?: string}>}
|
|
3190
|
+
*/
|
|
3191
|
+
_buildActionsFromPlan(plan) {
|
|
3192
|
+
const normalizedPlan = Array.isArray(plan)
|
|
3193
|
+
? { ordersToPlace: plan }
|
|
3194
|
+
: (plan || {});
|
|
3195
|
+
const { ordersToPlace = [], ordersToRotate = [], ordersToUpdate = [], ordersToCancel = [] } = normalizedPlan;
|
|
3196
|
+
const actions = [];
|
|
3197
|
+
for (const o of ordersToCancel) {
|
|
3198
|
+
if (o?.orderId) {
|
|
3199
|
+
actions.push({ type: COW_ACTIONS.CANCEL, id: o.id, orderId: o.orderId });
|
|
3200
|
+
}
|
|
3201
|
+
}
|
|
3202
|
+
for (const r of ordersToRotate) {
|
|
3203
|
+
const oldOrder = r?.oldOrder || r;
|
|
3204
|
+
const id = oldOrder?.id || r?.id;
|
|
3205
|
+
const orderId = oldOrder?.orderId || r?.orderId;
|
|
3206
|
+
const newGridId = r?.newGridId || id;
|
|
3207
|
+
const newSize = Number.isFinite(Number(r?.newSize))
|
|
3208
|
+
? Number(r.newSize)
|
|
3209
|
+
: Number(r?.size || oldOrder?.size || 0);
|
|
3210
|
+
const newPrice = Number.isFinite(Number(r?.newPrice))
|
|
3211
|
+
? Number(r.newPrice)
|
|
3212
|
+
: Number(r?.price || oldOrder?.price);
|
|
3213
|
+
const orderType = r?.type || oldOrder?.type;
|
|
3214
|
+
if (!id || !orderId || !newGridId || !orderType || !Number.isFinite(newPrice) || !(newSize > 0))
|
|
3215
|
+
continue;
|
|
3216
|
+
actions.push({
|
|
3217
|
+
type: COW_ACTIONS.UPDATE,
|
|
3218
|
+
id,
|
|
3219
|
+
orderId,
|
|
3220
|
+
newGridId,
|
|
3221
|
+
newSize,
|
|
3222
|
+
newPrice,
|
|
3223
|
+
order: {
|
|
3224
|
+
id: newGridId,
|
|
3225
|
+
type: orderType,
|
|
3226
|
+
price: newPrice,
|
|
3227
|
+
size: newSize
|
|
3228
|
+
}
|
|
3229
|
+
});
|
|
3230
|
+
}
|
|
3231
|
+
for (const o of ordersToUpdate) {
|
|
3232
|
+
const partialOrder = o?.partialOrder || o;
|
|
3233
|
+
const id = o?.id || partialOrder?.id;
|
|
3234
|
+
const orderId = o?.orderId || partialOrder?.orderId;
|
|
3235
|
+
const orderType = o?.type || partialOrder?.type;
|
|
3236
|
+
const newSize = Number.isFinite(Number(o?.newSize))
|
|
3237
|
+
? Number(o.newSize)
|
|
3238
|
+
: Number(partialOrder?.size || 0);
|
|
3239
|
+
if (!id || !orderId)
|
|
3240
|
+
continue;
|
|
3241
|
+
actions.push({
|
|
3242
|
+
type: COW_ACTIONS.UPDATE,
|
|
3243
|
+
id,
|
|
3244
|
+
orderId,
|
|
3245
|
+
newSize,
|
|
3246
|
+
order: {
|
|
3247
|
+
...(partialOrder || {}),
|
|
3248
|
+
id,
|
|
3249
|
+
orderId,
|
|
3250
|
+
type: orderType,
|
|
3251
|
+
size: newSize
|
|
3252
|
+
}
|
|
3253
|
+
});
|
|
3254
|
+
}
|
|
3255
|
+
// Run CREATE actions after UPDATE actions so same-batch downsizes can
|
|
3256
|
+
// release balance before placements consume it.
|
|
3257
|
+
for (const o of ordersToPlace) {
|
|
3258
|
+
if (!o?.id)
|
|
3259
|
+
continue;
|
|
3260
|
+
actions.push({ type: COW_ACTIONS.CREATE, id: o.id, order: o });
|
|
3261
|
+
}
|
|
3262
|
+
return actions;
|
|
3263
|
+
}
|
|
3264
|
+
/**
|
|
3265
|
+
* Build a COW result object (workingGrid + actions) from a simple plan.
|
|
3266
|
+
* @param {Object|Array} plan - Plan object or array of ordersToPlace
|
|
3267
|
+
* @returns {{workingGrid: import('./types').WorkingGrid, workingIndexes: Object, workingBoundary: number, actions: Array}}
|
|
3268
|
+
*/
|
|
3269
|
+
_buildCowResultFromPlan(plan) {
|
|
3270
|
+
const { WorkingGrid } = require('./order/working_grid');
|
|
3271
|
+
const workingGrid = new WorkingGrid(this.manager.orders, {
|
|
3272
|
+
baseVersion: Number.isFinite(Number(this.manager._gridVersion)) ? this.manager._gridVersion : 0
|
|
3273
|
+
});
|
|
3274
|
+
const workingBoundary = this.manager.boundaryIdx;
|
|
3275
|
+
const actions = this._buildActionsFromPlan(plan);
|
|
3276
|
+
// Project planned actions into working grid so COW commit carries intended transitions.
|
|
3277
|
+
for (const action of actions) {
|
|
3278
|
+
if (action.type === COW_ACTIONS.CANCEL) {
|
|
3279
|
+
const current = workingGrid.get(action.id);
|
|
3280
|
+
if (!current)
|
|
3281
|
+
continue;
|
|
3282
|
+
workingGrid.set(action.id, convertToSpreadPlaceholder(current));
|
|
3283
|
+
}
|
|
3284
|
+
else if (action.type === COW_ACTIONS.CREATE) {
|
|
3285
|
+
if (!action.id || !action.order)
|
|
3286
|
+
continue;
|
|
3287
|
+
const current = workingGrid.get(action.id) || { id: action.id };
|
|
3288
|
+
workingGrid.set(action.id, {
|
|
3289
|
+
...current,
|
|
3290
|
+
...action.order,
|
|
3291
|
+
id: action.id,
|
|
3292
|
+
state: ORDER_STATES.VIRTUAL,
|
|
3293
|
+
orderId: null
|
|
3294
|
+
});
|
|
3295
|
+
}
|
|
3296
|
+
else if (action.type === COW_ACTIONS.UPDATE) {
|
|
3297
|
+
if (action.newGridId && action.newGridId !== action.id) {
|
|
3298
|
+
const current = workingGrid.get(action.id);
|
|
3299
|
+
if (current) {
|
|
3300
|
+
workingGrid.set(action.id, convertToSpreadPlaceholder(current));
|
|
3301
|
+
}
|
|
3302
|
+
const targetId = action.newGridId;
|
|
3303
|
+
const targetCurrent = workingGrid.get(targetId) || { id: targetId };
|
|
3304
|
+
const rotatedSize = Number.isFinite(Number(action.newSize))
|
|
3305
|
+
? Number(action.newSize)
|
|
3306
|
+
: Number(targetCurrent.size || 0);
|
|
3307
|
+
const rotatedPrice = Number.isFinite(Number(action.newPrice))
|
|
3308
|
+
? Number(action.newPrice)
|
|
3309
|
+
: Number(action.order?.price ?? targetCurrent.price);
|
|
3310
|
+
workingGrid.set(targetId, {
|
|
3311
|
+
...targetCurrent,
|
|
3312
|
+
...(action.order || {}),
|
|
3313
|
+
id: targetId,
|
|
3314
|
+
size: rotatedSize,
|
|
3315
|
+
price: rotatedPrice,
|
|
3316
|
+
state: ORDER_STATES.VIRTUAL,
|
|
3317
|
+
orderId: null
|
|
3318
|
+
});
|
|
3319
|
+
continue;
|
|
3320
|
+
}
|
|
3321
|
+
const current = workingGrid.get(action.id);
|
|
3322
|
+
if (!current)
|
|
3323
|
+
continue;
|
|
3324
|
+
const newSize = Number.isFinite(Number(action.newSize))
|
|
3325
|
+
? Number(action.newSize)
|
|
3326
|
+
: Number(current.size || 0);
|
|
3327
|
+
workingGrid.set(action.id, {
|
|
3328
|
+
...current,
|
|
3329
|
+
...(action.order || {}),
|
|
3330
|
+
id: action.id,
|
|
3331
|
+
orderId: action.orderId || current.orderId,
|
|
3332
|
+
size: newSize
|
|
3333
|
+
});
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
return {
|
|
3337
|
+
workingGrid,
|
|
3338
|
+
workingIndexes: workingGrid.getIndexes(),
|
|
3339
|
+
workingBoundary,
|
|
3340
|
+
actions
|
|
3341
|
+
};
|
|
3342
|
+
}
|
|
3343
|
+
/**
|
|
3344
|
+
* Restore skipped update slots in the working grid to master state.
|
|
3345
|
+
* @param {import('./types').WorkingGrid} workingGrid - Working grid to restore slots into
|
|
3346
|
+
* @param {Set<string>} skippedSlotIds - Set of slot IDs that were skipped
|
|
3347
|
+
* @param {number} [skippedCount=0] - Count of skipped actions for logging
|
|
3348
|
+
* @returns {void}
|
|
3349
|
+
*/
|
|
3350
|
+
_restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedSlotIds, skippedCount = 0) {
|
|
3351
|
+
if (!workingGrid || !skippedSlotIds || skippedSlotIds.size === 0) {
|
|
3352
|
+
return;
|
|
3353
|
+
}
|
|
3354
|
+
const masterVersion = Number.isFinite(Number(this.manager?._gridVersion))
|
|
3355
|
+
? Number(this.manager._gridVersion)
|
|
3356
|
+
: undefined;
|
|
3357
|
+
for (const slotId of skippedSlotIds) {
|
|
3358
|
+
workingGrid.syncFromMaster(this.manager.orders, slotId, masterVersion);
|
|
3359
|
+
}
|
|
3360
|
+
this.manager.logger.log(`[COW] Restored ${skippedSlotIds.size} slot(s) after ${skippedCount} skipped update action(s).`, 'debug');
|
|
3361
|
+
}
|
|
3362
|
+
/**
|
|
3363
|
+
* COW broadcast: Execute blockchain operations and commit working grid on success.
|
|
3364
|
+
* Master grid is ONLY updated after successful blockchain confirmation.
|
|
3365
|
+
* @param {Object} cowResult - COW result with workingGrid, actions, etc.
|
|
3366
|
+
* @returns {Promise<Object>} The batch result.
|
|
3367
|
+
* @private
|
|
3368
|
+
*/
|
|
3369
|
+
async _updateOrdersOnChainBatchCOW(cowResult) {
|
|
3370
|
+
this._currentCycleId = (Number.isFinite(Number(this._currentCycleId)) ? Number(this._currentCycleId) : 0) + 1;
|
|
3371
|
+
const { workingGrid, workingIndexes, workingBoundary, actions } = cowResult;
|
|
3372
|
+
if (this.config.dryRun) {
|
|
3373
|
+
const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
|
|
3374
|
+
const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
|
|
3375
|
+
const updateCount = actions.filter(a => a.type === COW_ACTIONS.UPDATE).length;
|
|
3376
|
+
if (cancelCount > 0)
|
|
3377
|
+
this.manager.logger.log(`Dry run: would cancel ${cancelCount} orders`, 'info');
|
|
3378
|
+
if (createCount > 0)
|
|
3379
|
+
this.manager.logger.log(`Dry run: would place ${createCount} new orders`, 'info');
|
|
3380
|
+
if (updateCount > 0)
|
|
3381
|
+
this.manager.logger.log(`Dry run: would update ${updateCount} orders`, 'info');
|
|
3382
|
+
return { executed: true, hadRotation: false };
|
|
3383
|
+
}
|
|
3384
|
+
const createSlotValidation = validateCreateTargetSlots(actions, this.manager?.orders);
|
|
3385
|
+
if (!createSlotValidation.isValid) {
|
|
3386
|
+
for (const violation of createSlotValidation.violations) {
|
|
3387
|
+
this.manager.logger.log(`[COW] Rejecting CREATE for occupied slot ${violation.targetId}: ` +
|
|
3388
|
+
`existing orderId=${violation.currentOrderId}, type=${violation.currentType}, state=${violation.currentState}`, 'error');
|
|
3389
|
+
}
|
|
3390
|
+
return {
|
|
3391
|
+
executed: false,
|
|
3392
|
+
aborted: true,
|
|
3393
|
+
reason: 'CREATE_SLOT_OCCUPIED',
|
|
3394
|
+
violations: createSlotValidation.violations,
|
|
3395
|
+
hadRotation: false
|
|
3396
|
+
};
|
|
3397
|
+
}
|
|
3398
|
+
const hasCreateActions = actions.some(action => action.type === COW_ACTIONS.CREATE);
|
|
3399
|
+
const unmatchedChainOrders = Array.isArray(this.manager?._lastUnmatchedChainOrders)
|
|
3400
|
+
? this.manager._lastUnmatchedChainOrders
|
|
3401
|
+
: [];
|
|
3402
|
+
// PENDING_BROADCASTS: an earlier batch timed out at the credential
|
|
3403
|
+
// daemon and the chain status of its CREATEs is still unknown. The
|
|
3404
|
+
// recovery path (_reconcileAfterUncertainBroadcast) is responsible
|
|
3405
|
+
// for resolving it. We refuse to publish a fresh CREATE batch until
|
|
3406
|
+
// recovery runs, otherwise we risk double-publishing or stacking
|
|
3407
|
+
// orphan orders on top of potentially-orphaned ones.
|
|
3408
|
+
const pendingBroadcasts = (this.manager && this.manager._pendingBroadcasts instanceof Map)
|
|
3409
|
+
? Array.from(this.manager._pendingBroadcasts.values())
|
|
3410
|
+
: [];
|
|
3411
|
+
if (hasCreateActions && (unmatchedChainOrders.length > 0 || pendingBroadcasts.length > 0)) {
|
|
3412
|
+
const blockers = [];
|
|
3413
|
+
if (unmatchedChainOrders.length > 0)
|
|
3414
|
+
blockers.push(`${unmatchedChainOrders.length} unmatched chain order(s)`);
|
|
3415
|
+
if (pendingBroadcasts.length > 0)
|
|
3416
|
+
blockers.push(`${pendingBroadcasts.length} pending broadcast(s)`);
|
|
3417
|
+
const reasonText = blockers.join(' and ');
|
|
3418
|
+
if (pendingBroadcasts.length > 0) {
|
|
3419
|
+
this.manager.logger.log(`[COW] Rejecting CREATE batch: ${reasonText} from a prior uncertain ` +
|
|
3420
|
+
`broadcast. Running recovery before placing replacement orders.`, 'error');
|
|
3421
|
+
}
|
|
3422
|
+
else {
|
|
3423
|
+
const sample = unmatchedChainOrders
|
|
3424
|
+
.slice(0, 3)
|
|
3425
|
+
.map(order => this._formatUnmatchedChainOrderForLog(order))
|
|
3426
|
+
.join(' | ');
|
|
3427
|
+
this.manager.logger.log(`[COW] Rejecting CREATE batch: ${reasonText} ` +
|
|
3428
|
+
`are not represented in the grid${sample ? ` (${sample})` : ''}. ` +
|
|
3429
|
+
`Run structural reconciliation before placing replacement orders.`, 'error');
|
|
3430
|
+
}
|
|
3431
|
+
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
3432
|
+
if (this.manager._recoveryState)
|
|
3433
|
+
this.manager._recoveryState.structuralResyncRequested = true;
|
|
3434
|
+
await this.manager.requestStructuralGridResync(pendingBroadcasts.length > 0
|
|
3435
|
+
? 'pending broadcasts before COW create'
|
|
3436
|
+
: 'unmatched chain orders before COW create', pendingBroadcasts.length > 0
|
|
3437
|
+
? { pendingBroadcasts: pendingBroadcasts.map(p => p.slotId) }
|
|
3438
|
+
: { unmatchedChainOrders });
|
|
3439
|
+
}
|
|
3440
|
+
// If we have pending broadcasts, drive the recovery now so the
|
|
3441
|
+
// next planning cycle has a clean state.
|
|
3442
|
+
if (pendingBroadcasts.length > 0) {
|
|
3443
|
+
try {
|
|
3444
|
+
await this._reconcileAfterUncertainBroadcast(new BroadcastUncertainError('rejected CREATE batch had pending broadcasts', {
|
|
3445
|
+
operations: pendingBroadcasts.map(p => p.order),
|
|
3446
|
+
accountName: this.account,
|
|
3447
|
+
batchId: this._currentBatchId || null,
|
|
3448
|
+
payload: null,
|
|
3449
|
+
timeoutMs: null
|
|
3450
|
+
}), [], { fillLockAlreadyHeld: true });
|
|
3451
|
+
}
|
|
3452
|
+
catch (recoverErr) {
|
|
3453
|
+
this.manager.logger.log(`[COW] Recovery from pending broadcasts failed: ${recoverErr?.message || recoverErr}`, 'error');
|
|
3454
|
+
}
|
|
3455
|
+
}
|
|
3456
|
+
return {
|
|
3457
|
+
executed: false,
|
|
3458
|
+
aborted: true,
|
|
3459
|
+
reason: pendingBroadcasts.length > 0 ? 'PENDING_BROADCASTS' : 'UNMATCHED_CHAIN_ORDERS',
|
|
3460
|
+
unmatchedChainOrders: pendingBroadcasts.length > 0 ? [] : unmatchedChainOrders,
|
|
3461
|
+
pendingBroadcasts: pendingBroadcasts.map(p => p.slotId),
|
|
3462
|
+
hadRotation: false
|
|
3463
|
+
};
|
|
3464
|
+
}
|
|
3465
|
+
const { assetA, assetB } = this.manager.assets;
|
|
3466
|
+
const operations = [];
|
|
3467
|
+
const opContexts = [];
|
|
3468
|
+
const skippedUpdateSlotIds = new Set();
|
|
3469
|
+
let skippedUpdateCount = 0;
|
|
3470
|
+
// Collect IDs to lock from actions
|
|
3471
|
+
const idsToLock = new Set();
|
|
3472
|
+
for (const action of actions) {
|
|
3473
|
+
if (action.type === COW_ACTIONS.CANCEL && action.orderId) {
|
|
3474
|
+
idsToLock.add(action.orderId);
|
|
3475
|
+
if (action.id)
|
|
3476
|
+
idsToLock.add(action.id);
|
|
3477
|
+
}
|
|
3478
|
+
else if (action.type === COW_ACTIONS.CREATE && action.id) {
|
|
3479
|
+
idsToLock.add(action.id);
|
|
3480
|
+
}
|
|
3481
|
+
else if (action.type === COW_ACTIONS.UPDATE && action.orderId) {
|
|
3482
|
+
idsToLock.add(action.orderId);
|
|
3483
|
+
if (action.id)
|
|
3484
|
+
idsToLock.add(action.id);
|
|
3485
|
+
}
|
|
3486
|
+
}
|
|
3487
|
+
// Apply shadow locks
|
|
3488
|
+
this.manager.lockOrders(idsToLock);
|
|
3489
|
+
try {
|
|
3490
|
+
this._batchInFlight = true;
|
|
3491
|
+
this._markGridActivity('batch start');
|
|
3492
|
+
this.manager._setRebalanceState(REBALANCE_STATES.BROADCASTING);
|
|
3493
|
+
this.manager.startBroadcasting();
|
|
3494
|
+
// Build operations from actions
|
|
3495
|
+
for (const action of actions) {
|
|
3496
|
+
if (action.type === COW_ACTIONS.CANCEL) {
|
|
3497
|
+
try {
|
|
3498
|
+
const op = await chainOrders.buildCancelOrderOp(this.account, action.orderId);
|
|
3499
|
+
operations.push(op);
|
|
3500
|
+
const order = this.manager.orders.get(action.id) || { id: action.id, orderId: action.orderId };
|
|
3501
|
+
opContexts.push({ kind: 'cancel', order });
|
|
3502
|
+
}
|
|
3503
|
+
catch (err) {
|
|
3504
|
+
this.manager.logger.log(`Failed to prepare cancel op for ${action.id}: ${err.message}`, 'error');
|
|
3505
|
+
}
|
|
3506
|
+
}
|
|
3507
|
+
else if (action.type === COW_ACTIONS.CREATE) {
|
|
3508
|
+
try {
|
|
3509
|
+
const order = action.order;
|
|
3510
|
+
const sizeValidation = this._validateOrderSizeForExecution(order.size, order.type, order, order.size);
|
|
3511
|
+
if (!sizeValidation.isValid) {
|
|
3512
|
+
this.manager.logger.log(`Skipping create op for ${action.id}: ${sizeValidation.reason}`, 'warn');
|
|
3513
|
+
continue;
|
|
3514
|
+
}
|
|
3515
|
+
const liveSlot = this.manager.orders.get(order.id);
|
|
3516
|
+
const plannedPrice = Number(order.price);
|
|
3517
|
+
const livePrice = liveSlot ? Number(liveSlot.price) : NaN;
|
|
3518
|
+
const priceDrift = Number.isFinite(plannedPrice) && Number.isFinite(livePrice)
|
|
3519
|
+
? Math.abs(livePrice - plannedPrice)
|
|
3520
|
+
: 0;
|
|
3521
|
+
const effectiveOrder = (priceDrift > 0)
|
|
3522
|
+
? { ...order, price: livePrice, size: order.size, type: order.type }
|
|
3523
|
+
: order;
|
|
3524
|
+
if (priceDrift > 0) {
|
|
3525
|
+
this.manager.logger.log(`[COW] Pre-broadcast price freshness: slot ${order.id} ` +
|
|
3526
|
+
`drifted from planned=${plannedPrice} to live=${livePrice} ` +
|
|
3527
|
+
`(diff=${priceDrift}); rebuilding CREATE op with live price.`, 'debug');
|
|
3528
|
+
}
|
|
3529
|
+
const args = buildCreateOrderArgs(effectiveOrder, assetA, assetB);
|
|
3530
|
+
const buildResult = await chainOrders.buildCreateOrderOp(this.account, args.amountToSell, args.sellAssetId, args.minToReceive, args.receiveAssetId, null);
|
|
3531
|
+
if (!buildResult) {
|
|
3532
|
+
this.manager.logger.log(`Skipping create op for ${action.id}: amounts would round to 0 on blockchain`, 'warn');
|
|
3533
|
+
continue;
|
|
3534
|
+
}
|
|
3535
|
+
operations.push(buildResult.op);
|
|
3536
|
+
opContexts.push({ kind: 'create', id: order.id, order: effectiveOrder, args, finalInts: buildResult.finalInts });
|
|
3537
|
+
this._recordPendingBroadcast({
|
|
3538
|
+
opIndex: operations.length - 1,
|
|
3539
|
+
ctxIndex: opContexts.length - 1,
|
|
3540
|
+
order: effectiveOrder,
|
|
3541
|
+
finalInts: buildResult.finalInts
|
|
3542
|
+
});
|
|
3543
|
+
}
|
|
3544
|
+
catch (err) {
|
|
3545
|
+
this.manager.logger.log(`Failed to prepare create op for ${action.id}: ${err.message}`, 'error');
|
|
3546
|
+
}
|
|
3547
|
+
}
|
|
3548
|
+
else if (action.type === COW_ACTIONS.UPDATE) {
|
|
3549
|
+
try {
|
|
3550
|
+
// Rotation update: move existing on-chain order to a new slot/price.
|
|
3551
|
+
if (action.newGridId && action.newGridId !== action.id) {
|
|
3552
|
+
const masterOrder = this.manager.orders.get(action.id);
|
|
3553
|
+
const orderType = action.order?.type || masterOrder?.type;
|
|
3554
|
+
const newPrice = Number.isFinite(Number(action.newPrice))
|
|
3555
|
+
? Number(action.newPrice)
|
|
3556
|
+
: Number(action.order?.price);
|
|
3557
|
+
const newSize = Number.isFinite(Number(action.newSize))
|
|
3558
|
+
? Number(action.newSize)
|
|
3559
|
+
: Number(action.order?.size || 0);
|
|
3560
|
+
if (!masterOrder || !action.orderId || !orderType || !Number.isFinite(newPrice) || newSize <= 0) {
|
|
3561
|
+
continue;
|
|
3562
|
+
}
|
|
3563
|
+
const rotationSizeValidation = this._validateOrderSizeForExecution(newSize, orderType, action.order, newSize);
|
|
3564
|
+
if (!rotationSizeValidation.isValid) {
|
|
3565
|
+
this.manager.logger.log(`Skipping rotation update ${action.id} -> ${action.newGridId}: ${rotationSizeValidation.reason}`, 'warn');
|
|
3566
|
+
continue;
|
|
3567
|
+
}
|
|
3568
|
+
const { amountToSell, minToReceive } = buildCreateOrderArgs({ type: orderType, size: newSize, price: newPrice }, assetA, assetB);
|
|
3569
|
+
const buildResult = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell, minToReceive, newPrice, orderType }, masterOrder.rawOnChain || null);
|
|
3570
|
+
if (!buildResult) {
|
|
3571
|
+
skippedUpdateCount++;
|
|
3572
|
+
if (action.id)
|
|
3573
|
+
skippedUpdateSlotIds.add(action.id);
|
|
3574
|
+
if (action.newGridId)
|
|
3575
|
+
skippedUpdateSlotIds.add(action.newGridId);
|
|
3576
|
+
this.manager.logger.log(`[COW] Skipping rotation update ${action.id} -> ${action.newGridId}: no blockchain delta`, 'debug');
|
|
3577
|
+
continue;
|
|
3578
|
+
}
|
|
3579
|
+
operations.push(buildResult.op);
|
|
3580
|
+
opContexts.push({
|
|
3581
|
+
kind: 'rotation',
|
|
3582
|
+
rotation: {
|
|
3583
|
+
oldOrder: { ...masterOrder },
|
|
3584
|
+
newGridId: action.newGridId,
|
|
3585
|
+
newPrice,
|
|
3586
|
+
newSize,
|
|
3587
|
+
type: orderType
|
|
3588
|
+
},
|
|
3589
|
+
finalInts: buildResult.finalInts
|
|
3590
|
+
});
|
|
3591
|
+
continue;
|
|
3592
|
+
}
|
|
3593
|
+
const newSize = Number.isFinite(Number(action.newSize))
|
|
3594
|
+
? Number(action.newSize)
|
|
3595
|
+
: Number(action.order?.size || 0);
|
|
3596
|
+
const masterOrder = this.manager.orders.get(action.id);
|
|
3597
|
+
const orderType = action.order?.type || masterOrder?.type;
|
|
3598
|
+
const cachedRawOnChain = masterOrder?.rawOnChain || action.order?.rawOnChain || null;
|
|
3599
|
+
const op = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell: newSize, orderType }, cachedRawOnChain);
|
|
3600
|
+
if (!op) {
|
|
3601
|
+
skippedUpdateCount++;
|
|
3602
|
+
if (action.id)
|
|
3603
|
+
skippedUpdateSlotIds.add(action.id);
|
|
3604
|
+
if (action.newGridId)
|
|
3605
|
+
skippedUpdateSlotIds.add(action.newGridId);
|
|
3606
|
+
this.manager.logger.log(`[COW] Skipping size update ${action.id} (${action.orderId}): no blockchain delta`, 'debug');
|
|
3607
|
+
continue;
|
|
3608
|
+
}
|
|
3609
|
+
operations.push(op.op);
|
|
3610
|
+
const partialOrder = masterOrder || {
|
|
3611
|
+
id: action.id,
|
|
3612
|
+
orderId: action.orderId,
|
|
3613
|
+
type: orderType
|
|
3614
|
+
};
|
|
3615
|
+
opContexts.push({ kind: 'size-update', updateInfo: { partialOrder, newSize }, finalInts: op.finalInts });
|
|
3616
|
+
}
|
|
3617
|
+
catch (err) {
|
|
3618
|
+
this.manager.logger.log(`Failed to prepare update op for ${action.id}: ${err.message}`, 'error');
|
|
3619
|
+
}
|
|
3620
|
+
}
|
|
3621
|
+
}
|
|
3622
|
+
if (skippedUpdateCount > 0) {
|
|
3623
|
+
this._restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedUpdateSlotIds, skippedUpdateCount);
|
|
3624
|
+
}
|
|
3625
|
+
if (operations.length === 0) {
|
|
3626
|
+
this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
|
|
3627
|
+
return { executed: false, hadRotation: false };
|
|
3628
|
+
}
|
|
3629
|
+
// Validate funds before broadcasting
|
|
3630
|
+
const validation = this._validateOperationFunds(operations, assetA, assetB);
|
|
3631
|
+
this.manager.logger.log(validation.summary, validation.isValid ? 'info' : 'warn');
|
|
3632
|
+
if (!validation.isValid) {
|
|
3633
|
+
this.manager.logger.log(`Skipping batch broadcast: ${validation.violations.length} fund violation(s) detected`, 'warn');
|
|
3634
|
+
this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
|
|
3635
|
+
return { executed: false, hadRotation: false };
|
|
3636
|
+
}
|
|
3637
|
+
await this._ensureCredentialDaemonWritable('COW batch broadcast');
|
|
3638
|
+
// Execute batch
|
|
3639
|
+
this.manager.logger.log(`[COW] Broadcasting batch with ${operations.length} operations...`, 'info');
|
|
3640
|
+
this._lastBroadcastHeartbeatAt = Date.now();
|
|
3641
|
+
const execution = await this._executeWithRetryOnUncertain(operations, opContexts);
|
|
3642
|
+
const result = execution.result;
|
|
3643
|
+
const executedContexts = execution.opContexts;
|
|
3644
|
+
// Process results and commit on success
|
|
3645
|
+
this.manager.pauseFundRecalc();
|
|
3646
|
+
try {
|
|
3647
|
+
this.manager._throwOnIllegalState = true;
|
|
3648
|
+
if (result.success) {
|
|
3649
|
+
// Pre-commit integrity: only CREATE ops require returned chainOrderIds.
|
|
3650
|
+
// Cancel/update operation results may be empty depending on the broadcaster.
|
|
3651
|
+
const preCommitResults = this._extractOperationResults(result, 'pre-commit-integrity');
|
|
3652
|
+
const missingCreateResults = this._findMissingCreateResultContexts(preCommitResults, executedContexts);
|
|
3653
|
+
if (missingCreateResults.length > 0) {
|
|
3654
|
+
const missingSlots = missingCreateResults
|
|
3655
|
+
.map(item => item.ctx?.order?.id || item.ctx?.id || `op-${item.index}`)
|
|
3656
|
+
.join(', ');
|
|
3657
|
+
this.manager.logger.log(`[COW] Refusing to commit working grid: ${missingCreateResults.length} CREATE op(s) ` +
|
|
3658
|
+
`returned no chainOrderId (${missingSlots}). Discarding working grid and syncing from chain.`, 'error');
|
|
3659
|
+
this.manager._clearWorkingGridRef();
|
|
3660
|
+
this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
|
|
3661
|
+
this._markMissingCreateResultsAsStructuralBlocker(missingCreateResults);
|
|
3662
|
+
await this._recoverAfterMissingCreateResults('missing create operation results');
|
|
3663
|
+
return {
|
|
3664
|
+
executed: false,
|
|
3665
|
+
hadRotation: false,
|
|
3666
|
+
missingCreateResults: missingCreateResults.map(item => ({
|
|
3667
|
+
index: item.index,
|
|
3668
|
+
slotId: item.ctx?.order?.id || item.ctx?.id || null
|
|
3669
|
+
}))
|
|
3670
|
+
};
|
|
3671
|
+
}
|
|
3672
|
+
// SUCCESS: Commit working grid to master (atomic swap)
|
|
3673
|
+
this.manager.logger.log('[COW] Blockchain success - committing working grid to master', 'info');
|
|
3674
|
+
// RC-FIX: skipRecalc prevents invariant violation before optimistic accounting
|
|
3675
|
+
await this.manager._commitWorkingGrid(workingGrid, workingIndexes, workingBoundary, { skipRecalc: true });
|
|
3676
|
+
// Commitment accounting is handled in real-time by
|
|
3677
|
+
// updateOptimisticFreeBalance when capital is committed to orders.
|
|
3678
|
+
// The old post-batch deduction path was removed to avoid double-counting.
|
|
3679
|
+
// Process batch results for logging/metrics
|
|
3680
|
+
const batchResult = await this._processBatchResults(result, executedContexts);
|
|
3681
|
+
// Persist to disk. CRITICAL: working grid was already committed
|
|
3682
|
+
// to master above; if persistence is skipped or validation fails
|
|
3683
|
+
// here, the on-disk snapshot will be older than the in-memory
|
|
3684
|
+
// master. Retry once and surface the failure explicitly so the
|
|
3685
|
+
// next cycle / shutdown can recover.
|
|
3686
|
+
const persistResult = await this.manager.persistGrid();
|
|
3687
|
+
if (persistResult && (persistResult.skipped || persistResult.isValid === false)) {
|
|
3688
|
+
this.manager.logger.log(`[COW][PERSIST-GUARD] First persist attempt was ` +
|
|
3689
|
+
`${persistResult.skipped ? 'skipped' : 'invalid'} ` +
|
|
3690
|
+
`(${persistResult.reason || 'no reason'}); retrying once before ` +
|
|
3691
|
+
`clearing working grid reference.`, 'warn');
|
|
3692
|
+
this.manager._persistenceWarning = persistResult;
|
|
3693
|
+
const retryResult = await this.manager.persistGrid();
|
|
3694
|
+
if (retryResult && (retryResult.skipped || retryResult.isValid === false)) {
|
|
3695
|
+
this.manager.logger.log(`[COW][PERSIST-GUARD] Retry also skipped/invalid ` +
|
|
3696
|
+
`(${retryResult.reason || 'no reason'}). Master grid in memory ` +
|
|
3697
|
+
`is ahead of disk snapshot; structural resync requested.`, 'error');
|
|
3698
|
+
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
3699
|
+
this.manager._recoveryState = this.manager._recoveryState || {};
|
|
3700
|
+
this.manager._recoveryState.structuralResyncRequested = true;
|
|
3701
|
+
await this.manager.requestStructuralGridResync('persistence guard triggered after COW batch', { persistReason: retryResult.reason || 'unknown' });
|
|
3702
|
+
}
|
|
3703
|
+
}
|
|
3704
|
+
else {
|
|
3705
|
+
delete this.manager._persistenceWarning;
|
|
3706
|
+
}
|
|
3707
|
+
}
|
|
3708
|
+
else if (this.manager._persistenceWarning) {
|
|
3709
|
+
delete this.manager._persistenceWarning;
|
|
3710
|
+
}
|
|
3711
|
+
this._metrics.batchesExecuted++;
|
|
3712
|
+
this.manager._clearWorkingGridRef();
|
|
3713
|
+
this._clearPendingBroadcasts();
|
|
3714
|
+
return { executed: true, hadRotation: true, ...batchResult };
|
|
3715
|
+
}
|
|
3716
|
+
else {
|
|
3717
|
+
// FAILURE: Working grid discarded, master unchanged
|
|
3718
|
+
this.manager.logger.log('[COW] Blockchain failed - working grid discarded, master unchanged', 'warn');
|
|
3719
|
+
this.manager._clearWorkingGridRef();
|
|
3720
|
+
this._clearPendingBroadcasts();
|
|
3721
|
+
return { executed: false, hadRotation: false, ...result };
|
|
3722
|
+
}
|
|
3723
|
+
}
|
|
3724
|
+
finally {
|
|
3725
|
+
this.manager._throwOnIllegalState = false;
|
|
3726
|
+
// Keep broadcasting true during resumeFundRecalc to skip invariant checks
|
|
3727
|
+
// that would fail due to stale accountTotals (not yet refreshed from blockchain)
|
|
3728
|
+
await this.manager.resumeFundRecalc();
|
|
3729
|
+
this.manager.stopBroadcasting();
|
|
3730
|
+
const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
|
|
3731
|
+
const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
|
|
3732
|
+
this.manager.logger.logFundsStatus(this.manager, `AFTER COW batch (created=${createCount}, cancelled=${cancelCount})`);
|
|
3733
|
+
}
|
|
3734
|
+
}
|
|
3735
|
+
catch (err) {
|
|
3736
|
+
this.manager.logger.log(`[COW] Batch transaction failed: ${err.message}`, 'error');
|
|
3737
|
+
if (err?.partialOnChainState) {
|
|
3738
|
+
this.manager.logger.log(`[COW] Non-atomic grouped execution detected (${err.groupsBroadcast}/${err.groupsTotal} groups broadcast). Local rollback cannot undo confirmed on-chain operations; next sync/reconcile will converge state.`, 'warn');
|
|
3739
|
+
}
|
|
3740
|
+
this.manager.stopBroadcasting();
|
|
3741
|
+
this.manager._clearWorkingGridRef();
|
|
3742
|
+
// BROADCAST_UNCERTAIN: the credential daemon timed out (or hit its
|
|
3743
|
+
// inner deadline) and the chain status of the planned CREATEs is
|
|
3744
|
+
// unknown. Run the reconcile-then-decide recovery path: read the
|
|
3745
|
+
// chain, match each pending broadcast by fingerprint, adopt any
|
|
3746
|
+
// chain-side matches, and discard the rest. We MUST NOT throw —
|
|
3747
|
+
// throwing would re-enter the catch loop on the next attempt and
|
|
3748
|
+
// potentially double-publish.
|
|
3749
|
+
if (err instanceof BroadcastUncertainError) {
|
|
3750
|
+
return await this._reconcileAfterUncertainBroadcast(err, opContexts, { fillLockAlreadyHeld: true });
|
|
3751
|
+
}
|
|
3752
|
+
// Handle hard abort
|
|
3753
|
+
const hardAbortResult = await this._handleBatchHardAbort(err, 'COW batch processing', operations.length);
|
|
3754
|
+
if (hardAbortResult)
|
|
3755
|
+
return hardAbortResult;
|
|
3756
|
+
// Check for stale orders — filled in the ~1.5s broadcast window after our plan was built.
|
|
3757
|
+
const staleOrderIds = new Set();
|
|
3758
|
+
const patterns = [
|
|
3759
|
+
/Limit order (1\.7\.\d+) does not exist/g,
|
|
3760
|
+
/Unable to find Object (1\.7\.\d+)/g,
|
|
3761
|
+
/object (1\.7\.\d+) (?:does not exist|not found)/gi
|
|
3762
|
+
];
|
|
3763
|
+
for (const pattern of patterns) {
|
|
3764
|
+
let m;
|
|
3765
|
+
while ((m = pattern.exec(err.message)) !== null) {
|
|
3766
|
+
staleOrderIds.add(m[1]);
|
|
3767
|
+
}
|
|
3768
|
+
}
|
|
3769
|
+
// "Cannot deduct all or more from order than order contains" means the order still
|
|
3770
|
+
// exists, but its on-chain size shrank during the broadcast window. That is not an
|
|
3771
|
+
// explicit stale/missing-order signal, so reconcile from chain instead of virtualizing.
|
|
3772
|
+
if (/Cannot deduct all or more from order than order contains/.test(err.message)) {
|
|
3773
|
+
return await this._recoverBatchSizeDrift(err, opContexts);
|
|
3774
|
+
}
|
|
3775
|
+
if (staleOrderIds.size > 0) {
|
|
3776
|
+
// Recover explicit missing-order failures without aborting the entire fill cycle.
|
|
3777
|
+
// Use the manager update path so indexes, working-grid sync, accounting, and
|
|
3778
|
+
// persistence stay coherent.
|
|
3779
|
+
return await this._recoverExplicitStaleOrders(staleOrderIds, 'cow-stale-order-cleanup');
|
|
3780
|
+
}
|
|
3781
|
+
throw err;
|
|
3782
|
+
}
|
|
3783
|
+
finally {
|
|
3784
|
+
this._batchInFlight = false;
|
|
3785
|
+
this._markGridActivity('batch end');
|
|
3786
|
+
this.manager.unlockOrders(idsToLock);
|
|
3787
|
+
if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
|
|
3788
|
+
this._scheduleFillConsumerRestart(chainOrders);
|
|
3789
|
+
}
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3792
|
+
/**
|
|
3793
|
+
* Process results from batch transaction execution.
|
|
3794
|
+
* Updates order state, synchronizes with chain, and deducts BTS fees.
|
|
3795
|
+
* @param {Object} result - Transaction result from executeBatch
|
|
3796
|
+
* @param {Array} opContexts - Operation context array with operation metadata (must be 1:1 with result.operation_results)
|
|
3797
|
+
* @returns {Object} Result with { executed: boolean, hadRotation: boolean }
|
|
3798
|
+
* @private
|
|
3799
|
+
*/
|
|
3800
|
+
async _processBatchResults(result, opContexts) {
|
|
3801
|
+
const results = this._extractOperationResults(result, '_processBatchResults');
|
|
3802
|
+
const { getAssetFees } = require('./order/utils/math');
|
|
3803
|
+
// IMPORTANT: Call without amount to get fee schedule fields
|
|
3804
|
+
// ({ createFee, updateFee, ... }), not proceeds projection fields.
|
|
3805
|
+
const btsFeeData = getAssetFees('BTS');
|
|
3806
|
+
let hadRotation = false;
|
|
3807
|
+
let updateOperationCount = 0;
|
|
3808
|
+
const updatesToApply = [];
|
|
3809
|
+
for (let i = 0; i < opContexts.length; i++) {
|
|
3810
|
+
const ctx = opContexts[i];
|
|
3811
|
+
const res = results[i];
|
|
3812
|
+
if (ctx.kind === 'cancel') {
|
|
3813
|
+
this.manager.logger.log(`Cancelled surplus order ${ctx.order.id} (${ctx.order.orderId})`, 'info');
|
|
3814
|
+
const oldOrder = ctx.order;
|
|
3815
|
+
const committedOrder = oldOrder?.id ? this.manager.orders.get(oldOrder.id) : null;
|
|
3816
|
+
// The COW commit already updated manager.orders. Apply ONLY the optimistic
|
|
3817
|
+
// accounting transition using pre-commit -> committed order states.
|
|
3818
|
+
if (oldOrder && committedOrder && this.manager.accountant) {
|
|
3819
|
+
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, committedOrder, 'fill-cancel', btsFeeData?.cancelFee || 0, false);
|
|
3820
|
+
}
|
|
3821
|
+
}
|
|
3822
|
+
else if (ctx.kind === 'size-update') {
|
|
3823
|
+
const oldOrder = ctx.updateInfo.partialOrder;
|
|
3824
|
+
const ord = this.manager.orders.get(oldOrder.id);
|
|
3825
|
+
// Apply optimistic accounting from pre-commit -> committed state,
|
|
3826
|
+
// including blockchain update fee deduction.
|
|
3827
|
+
if (oldOrder && ord && this.manager.accountant) {
|
|
3828
|
+
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
|
|
3829
|
+
}
|
|
3830
|
+
if (ord) {
|
|
3831
|
+
const updatedSlot = { ...ord, size: ctx.updateInfo.newSize };
|
|
3832
|
+
// Update rawOnChain cache with new integers
|
|
3833
|
+
if (ctx.finalInts) {
|
|
3834
|
+
updatedSlot.rawOnChain = {
|
|
3835
|
+
id: ord.orderId,
|
|
3836
|
+
for_sale: String(ctx.finalInts.sell),
|
|
3837
|
+
sell_price: {
|
|
3838
|
+
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
3839
|
+
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
3840
|
+
}
|
|
3841
|
+
};
|
|
3842
|
+
}
|
|
3843
|
+
updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
|
|
3844
|
+
}
|
|
3845
|
+
this.manager.logger.log(`Size update complete: ${ctx.updateInfo.partialOrder.orderId}`, 'info');
|
|
3846
|
+
updateOperationCount++;
|
|
3847
|
+
}
|
|
3848
|
+
else if (ctx.kind === 'create') {
|
|
3849
|
+
const chainOrderId = res && res[1];
|
|
3850
|
+
if (chainOrderId) {
|
|
3851
|
+
// synchronizeWithChain handles the full VIRTUAL -> ACTIVE transition
|
|
3852
|
+
// including orderId assignment and fee deduction.
|
|
3853
|
+
await this.manager.synchronizeWithChain({
|
|
3854
|
+
gridOrderId: ctx.order.id, chainOrderId, expectedType: ctx.order.type, fee: btsFeeData.createFee
|
|
3855
|
+
}, 'createOrder');
|
|
3856
|
+
// After sync, apply rawOnChain metadata if available
|
|
3857
|
+
if (ctx.finalInts) {
|
|
3858
|
+
const syncedOrder = this.manager.orders.get(ctx.order.id);
|
|
3859
|
+
if (syncedOrder) {
|
|
3860
|
+
updatesToApply.push({
|
|
3861
|
+
order: {
|
|
3862
|
+
...syncedOrder,
|
|
3863
|
+
rawOnChain: {
|
|
3864
|
+
id: chainOrderId,
|
|
3865
|
+
for_sale: String(ctx.finalInts.sell),
|
|
3866
|
+
sell_price: {
|
|
3867
|
+
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
3868
|
+
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
3869
|
+
}
|
|
3870
|
+
}
|
|
3871
|
+
},
|
|
3872
|
+
context: 'post-placement-metadata'
|
|
3873
|
+
});
|
|
3874
|
+
}
|
|
3875
|
+
}
|
|
3876
|
+
this.manager.logger.log(`Placed ${ctx.order.type} order ${ctx.order.id} -> ${chainOrderId}`, 'info');
|
|
3877
|
+
}
|
|
3878
|
+
else {
|
|
3879
|
+
const fingerprint = [
|
|
3880
|
+
`type=${ctx.order.type || 'unknown'}`,
|
|
3881
|
+
`price=${Format.formatPrice6(ctx.order.price)}`,
|
|
3882
|
+
`size=${Format.formatAmount(ctx.order.size)}`
|
|
3883
|
+
].join(',');
|
|
3884
|
+
this.manager.logger.log(`[COW] CRITICAL: Create op for slot ${ctx.order.id} (type=${ctx.order.type}) ` +
|
|
3885
|
+
`returned no chainOrderId. Identify any orphaned on-chain order by local fingerprint ` +
|
|
3886
|
+
`${fingerprint} before cancelling.`, 'error');
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
else if (ctx.kind === 'rotation') {
|
|
3890
|
+
hadRotation = true;
|
|
3891
|
+
const { rotation } = ctx;
|
|
3892
|
+
const { oldOrder, newPrice, newGridId, newSize, type } = rotation;
|
|
3893
|
+
if (!newGridId) {
|
|
3894
|
+
// Size correction only
|
|
3895
|
+
const ord = this.manager.orders.get(oldOrder.id || rotation.id);
|
|
3896
|
+
// Apply optimistic accounting from pre-commit -> committed state,
|
|
3897
|
+
// including blockchain update fee deduction.
|
|
3898
|
+
if (oldOrder && ord && this.manager.accountant) {
|
|
3899
|
+
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
|
|
3900
|
+
}
|
|
3901
|
+
if (ord) {
|
|
3902
|
+
const updatedSlot = { ...ord, size: newSize };
|
|
3903
|
+
// Update rawOnChain cache with new integers
|
|
3904
|
+
if (ctx.finalInts) {
|
|
3905
|
+
updatedSlot.rawOnChain = {
|
|
3906
|
+
id: ord.orderId,
|
|
3907
|
+
for_sale: String(ctx.finalInts.sell),
|
|
3908
|
+
sell_price: {
|
|
3909
|
+
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
3910
|
+
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
3911
|
+
}
|
|
3912
|
+
};
|
|
3913
|
+
}
|
|
3914
|
+
updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
|
|
3915
|
+
}
|
|
3916
|
+
updateOperationCount++;
|
|
3917
|
+
continue;
|
|
3918
|
+
}
|
|
3919
|
+
// Full rotation: old slot was virtualized in the committed working grid.
|
|
3920
|
+
// Activate the destination slot with the existing on-chain orderId.
|
|
3921
|
+
const slot = this.manager.orders.get(newGridId);
|
|
3922
|
+
if (!slot) {
|
|
3923
|
+
this.manager.logger.log(`[ROTATION] Destination slot ${newGridId} missing from master grid after COW commit - skipping activation, sync will reconcile`, 'error');
|
|
3924
|
+
// Still clear source orderId to prevent a stale chain reference persisting
|
|
3925
|
+
if (oldOrder?.id && oldOrder.id !== newGridId) {
|
|
3926
|
+
const staleSource = this.manager.orders.get(oldOrder.id);
|
|
3927
|
+
if (staleSource?.orderId) {
|
|
3928
|
+
updatesToApply.push({
|
|
3929
|
+
order: { ...staleSource, state: ORDER_STATES.VIRTUAL, orderId: null, rawOnChain: null },
|
|
3930
|
+
context: 'post-rotation-source-clear'
|
|
3931
|
+
});
|
|
3932
|
+
}
|
|
3933
|
+
}
|
|
3934
|
+
continue;
|
|
3935
|
+
}
|
|
3936
|
+
const updatedSlot = {
|
|
3937
|
+
...slot,
|
|
3938
|
+
id: newGridId,
|
|
3939
|
+
type,
|
|
3940
|
+
size: newSize,
|
|
3941
|
+
price: newPrice,
|
|
3942
|
+
state: ORDER_STATES.ACTIVE,
|
|
3943
|
+
orderId: oldOrder?.orderId || slot.orderId || null
|
|
3944
|
+
};
|
|
3945
|
+
if (ctx.finalInts) {
|
|
3946
|
+
updatedSlot.rawOnChain = {
|
|
3947
|
+
id: updatedSlot.orderId,
|
|
3948
|
+
for_sale: String(ctx.finalInts.sell),
|
|
3949
|
+
sell_price: {
|
|
3950
|
+
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
3951
|
+
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
3952
|
+
}
|
|
3953
|
+
};
|
|
3954
|
+
}
|
|
3955
|
+
if (oldOrder && updatedSlot && this.manager.accountant) {
|
|
3956
|
+
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, updatedSlot, 'order-update', btsFeeData.updateFee, false);
|
|
3957
|
+
}
|
|
3958
|
+
// Ensure rotation source slot is cleared in master state.
|
|
3959
|
+
// COW projection may keep source slots ACTIVE when target keeps a non-zero
|
|
3960
|
+
// virtual size, but after a successful on-chain rotation the source orderId
|
|
3961
|
+
// must no longer remain attached to the old slot.
|
|
3962
|
+
if (oldOrder?.id && oldOrder.id !== newGridId) {
|
|
3963
|
+
const currentSource = this.manager.orders.get(oldOrder.id);
|
|
3964
|
+
if (currentSource && currentSource.orderId) {
|
|
3965
|
+
updatesToApply.push({
|
|
3966
|
+
order: {
|
|
3967
|
+
...currentSource,
|
|
3968
|
+
state: ORDER_STATES.VIRTUAL,
|
|
3969
|
+
orderId: null,
|
|
3970
|
+
rawOnChain: null
|
|
3971
|
+
},
|
|
3972
|
+
context: 'post-rotation-source-clear'
|
|
3973
|
+
});
|
|
3974
|
+
}
|
|
3975
|
+
}
|
|
3976
|
+
updatesToApply.push({ order: updatedSlot, context: 'post-rotation-metadata' });
|
|
3977
|
+
}
|
|
3978
|
+
}
|
|
3979
|
+
// Apply all collected updates in a single batch
|
|
3980
|
+
if (updatesToApply.length > 0) {
|
|
3981
|
+
await this.manager.applyGridUpdateBatch(updatesToApply.map(u => u.order), 'batch-results-process', { skipAccounting: true });
|
|
3982
|
+
}
|
|
3983
|
+
return {
|
|
3984
|
+
executed: true,
|
|
3985
|
+
hadRotation,
|
|
3986
|
+
updateOperationCount
|
|
3987
|
+
};
|
|
3988
|
+
}
|
|
3989
|
+
/**
|
|
3990
|
+
* Perform grid recalculation triggered by trigger file.
|
|
3991
|
+
* Reloads config from disk, recalculates grid, resets funds, and removes trigger file.
|
|
3992
|
+
* Must be called with _fillProcessingLock already held.
|
|
3993
|
+
* @param {Object} [options] - Optional configuration for grid resync.
|
|
3994
|
+
* @returns {Promise<boolean>} True if resync succeeded
|
|
3995
|
+
* @private
|
|
3996
|
+
*/
|
|
3997
|
+
async _performGridResync(options = {}) {
|
|
3998
|
+
return DexbotMaintenanceRuntime.performGridResync(this, options);
|
|
3999
|
+
}
|
|
4000
|
+
/**
|
|
4001
|
+
* Handle any pending trigger file reset at startup.
|
|
4002
|
+
* This is called FIRST during startup before any grid operations.
|
|
4003
|
+
* @returns {Promise<boolean>} True if trigger reset completed successfully, false otherwise
|
|
4004
|
+
* @private
|
|
4005
|
+
*/
|
|
4006
|
+
async _handlePendingTriggerReset() {
|
|
4007
|
+
return DexbotMaintenanceRuntime.handlePendingTriggerReset(this);
|
|
4008
|
+
}
|
|
4009
|
+
/**
|
|
4010
|
+
* Setup trigger file detection for grid reset.
|
|
4011
|
+
* Monitors the trigger file and performs grid resync when it's created.
|
|
4012
|
+
* @private
|
|
4013
|
+
*/
|
|
4014
|
+
async _setupTriggerFileDetection() {
|
|
4015
|
+
return DexbotMaintenanceRuntime.setupTriggerFileDetection(this);
|
|
4016
|
+
}
|
|
4017
|
+
/**
|
|
4018
|
+
* Starts the bot's operation.
|
|
4019
|
+
* @param {string|Object|Buffer} [vaultSecret=null] - The unlock secret.
|
|
4020
|
+
* @returns {Promise<void>}
|
|
4021
|
+
*/
|
|
4022
|
+
async start(vaultSecret = null) {
|
|
4023
|
+
await this.initialize(vaultSecret);
|
|
4024
|
+
await this._runStartupSequence();
|
|
4025
|
+
}
|
|
4026
|
+
/**
|
|
4027
|
+
* Start bot with a pre-decrypted private key.
|
|
4028
|
+
* Alternative to start(vaultSecret) when the signing secret is already available.
|
|
4029
|
+
* @param {string|Object} privateKey - Pre-decrypted private key or daemon signing token
|
|
4030
|
+
* @returns {Promise<void>}
|
|
4031
|
+
*/
|
|
4032
|
+
async startWithPrivateKey(privateKey) {
|
|
4033
|
+
// Initialize account data with provided private key
|
|
4034
|
+
await waitForConnected(TIMING.CONNECTION_TIMEOUT_MS);
|
|
4035
|
+
if (this.config && this.config.preferredAccount) {
|
|
4036
|
+
try {
|
|
4037
|
+
this.privateKey = privateKey;
|
|
4038
|
+
await this._setupAccountContext(this.config.preferredAccount);
|
|
4039
|
+
}
|
|
4040
|
+
catch (err) {
|
|
4041
|
+
this._warn(`Auto-selection of preferredAccount failed: ${err.message}`);
|
|
4042
|
+
throw err;
|
|
4043
|
+
}
|
|
4044
|
+
}
|
|
4045
|
+
else {
|
|
4046
|
+
throw new Error('No preferredAccount configured');
|
|
4047
|
+
}
|
|
4048
|
+
await this._runStartupSequence();
|
|
4049
|
+
}
|
|
4050
|
+
/**
|
|
4051
|
+
* Common startup sequence logic shared between start() and startWithPrivateKey().
|
|
4052
|
+
* @private
|
|
4053
|
+
*/
|
|
4054
|
+
async _runStartupSequence() {
|
|
4055
|
+
try {
|
|
4056
|
+
const startupState = await this._initializeStartupState();
|
|
4057
|
+
await this._finishStartupSequence(startupState);
|
|
4058
|
+
}
|
|
4059
|
+
catch (err) {
|
|
4060
|
+
this._warn(`Error during grid initialization: ${err.message}`);
|
|
4061
|
+
await this.shutdown();
|
|
4062
|
+
throw err;
|
|
4063
|
+
}
|
|
4064
|
+
}
|
|
4065
|
+
/**
|
|
4066
|
+
* Perform periodic grid checks: fund thresholds, spread condition, grid health.
|
|
4067
|
+
* Called by the periodic blockchain fetch interval to check if grid needs updates.
|
|
4068
|
+
*
|
|
4069
|
+
* IMPORTANT: This method MUST only be called from within _fillProcessingLock.acquire()
|
|
4070
|
+
* (specifically from _setupBlockchainFetchInterval). It passes fillLockAlreadyHeld
|
|
4071
|
+
* to avoid deadlock with _consumeFillQueue which uses the same lock ordering.
|
|
4072
|
+
*
|
|
4073
|
+
* @private
|
|
4074
|
+
*/
|
|
4075
|
+
async _performPeriodicGridChecks() {
|
|
4076
|
+
return DexbotMaintenanceRuntime.performPeriodicGridChecks(this);
|
|
4077
|
+
}
|
|
4078
|
+
_isOpenOrdersSyncLoopEnabled() {
|
|
4079
|
+
return DexbotMaintenanceRuntime.isOpenOrdersSyncLoopEnabled(this);
|
|
4080
|
+
}
|
|
4081
|
+
/**
|
|
4082
|
+
* Start the open-orders watchdog sync loop.
|
|
4083
|
+
* Uses fill lock contention checks to avoid competing with fill processing.
|
|
4084
|
+
* @private
|
|
4085
|
+
*/
|
|
4086
|
+
_startOpenOrdersSyncLoop() {
|
|
4087
|
+
return DexbotMaintenanceRuntime.startOpenOrdersSyncLoop(this);
|
|
4088
|
+
}
|
|
4089
|
+
/**
|
|
4090
|
+
* Stop the open-orders watchdog sync loop.
|
|
4091
|
+
* @private
|
|
4092
|
+
*/
|
|
4093
|
+
async _stopOpenOrdersSyncLoop() {
|
|
4094
|
+
return DexbotMaintenanceRuntime.stopOpenOrdersSyncLoop(this);
|
|
4095
|
+
}
|
|
4096
|
+
/**
|
|
4097
|
+
* Set up periodic blockchain account balance fetch interval.
|
|
4098
|
+
* Fetches available funds at regular intervals to keep blockchain variables up-to-date.
|
|
4099
|
+
* @private
|
|
4100
|
+
*/
|
|
4101
|
+
_setupBlockchainFetchInterval() {
|
|
4102
|
+
return DexbotMaintenanceRuntime.setupBlockchainFetchInterval(this);
|
|
4103
|
+
}
|
|
4104
|
+
/**
|
|
4105
|
+
* Stop the periodic blockchain fetch interval.
|
|
4106
|
+
* @private
|
|
4107
|
+
*/
|
|
4108
|
+
_stopBlockchainFetchInterval() {
|
|
4109
|
+
return DexbotMaintenanceRuntime.stopBlockchainFetchInterval(this);
|
|
4110
|
+
}
|
|
4111
|
+
async _releaseMarketAdapterRuntime(context = 'shutdown') {
|
|
4112
|
+
return DexbotMaintenanceRuntime.releaseMarketAdapterRuntime(this, this.config?.botKey || this.config?.name, context);
|
|
4113
|
+
}
|
|
4114
|
+
/**
|
|
4115
|
+
* Get or create the credit runtime for debt policy management.
|
|
4116
|
+
* @returns {import('./credit_runtime').CreditRuntime|null}
|
|
4117
|
+
*/
|
|
4118
|
+
_getCreditRuntime() {
|
|
4119
|
+
const lending = this.config?.debtPolicy?.lending;
|
|
4120
|
+
const enabledPolicy = Array.isArray(lending)
|
|
4121
|
+
&& lending.length > 0
|
|
4122
|
+
&& lending.every((item) => typeof item?.collateralAsset === 'string' && item.collateralAsset.length > 0);
|
|
4123
|
+
if (!enabledPolicy) {
|
|
4124
|
+
this._creditRuntime = null;
|
|
4125
|
+
return null;
|
|
4126
|
+
}
|
|
4127
|
+
if (!this._creditRuntime) {
|
|
4128
|
+
this._creditRuntime = new CreditRuntime(this, {
|
|
4129
|
+
stateDir: PATHS.CREDIT_RUNTIME_DIR,
|
|
4130
|
+
});
|
|
4131
|
+
}
|
|
4132
|
+
return this._creditRuntime;
|
|
4133
|
+
}
|
|
4134
|
+
/**
|
|
4135
|
+
* Set up the credit runtime by loading its persisted state.
|
|
4136
|
+
* @returns {Promise<import('./credit_runtime').CreditRuntime|null>}
|
|
4137
|
+
*/
|
|
4138
|
+
async _setupCreditRuntime() {
|
|
4139
|
+
const runtime = this._getCreditRuntime();
|
|
4140
|
+
if (!runtime) {
|
|
4141
|
+
return null;
|
|
4142
|
+
}
|
|
4143
|
+
await runtime.loadState();
|
|
4144
|
+
return runtime;
|
|
4145
|
+
}
|
|
4146
|
+
/**
|
|
4147
|
+
* Refresh credit runtime state from chain and sync internal tracking.
|
|
4148
|
+
* @returns {Promise<void>}
|
|
4149
|
+
*/
|
|
4150
|
+
async _refreshAndSyncCreditRuntime() {
|
|
4151
|
+
const runtime = this._getCreditRuntime();
|
|
4152
|
+
if (!runtime)
|
|
4153
|
+
return;
|
|
4154
|
+
try {
|
|
4155
|
+
await runtime.refreshState();
|
|
4156
|
+
}
|
|
4157
|
+
catch (err) {
|
|
4158
|
+
this._warn(`Credit runtime refresh/sync failed: ${err.message}`);
|
|
4159
|
+
}
|
|
4160
|
+
}
|
|
4161
|
+
/**
|
|
4162
|
+
* Run credit runtime maintenance (deal checks, collateral monitoring).
|
|
4163
|
+
* @param {string} [context='periodic'] - Maintenance context
|
|
4164
|
+
* @param {Object} [options={}] - Maintenance options
|
|
4165
|
+
* @returns {Promise<*>} Maintenance result from runtime
|
|
4166
|
+
*/
|
|
4167
|
+
async _runCreditRuntimeMaintenance(context = 'periodic', options = {}) {
|
|
4168
|
+
const runtime = this._getCreditRuntime();
|
|
4169
|
+
if (!runtime) {
|
|
4170
|
+
return null;
|
|
4171
|
+
}
|
|
4172
|
+
return runtime.runMaintenance(context, options);
|
|
4173
|
+
}
|
|
4174
|
+
/**
|
|
4175
|
+
* Start the credit deal watchdog interval.
|
|
4176
|
+
* @returns {void}
|
|
4177
|
+
*/
|
|
4178
|
+
_setupCreditWatchdogInterval() {
|
|
4179
|
+
const runtime = this._getCreditRuntime();
|
|
4180
|
+
if (!runtime) {
|
|
4181
|
+
return;
|
|
4182
|
+
}
|
|
4183
|
+
const intervalMin = Number(this.config?.TIMING?.CREDIT_DEAL_CHECK_INTERVAL_MIN ?? TIMING.CREDIT_DEAL_CHECK_INTERVAL_MIN);
|
|
4184
|
+
if (!Number.isFinite(intervalMin) || intervalMin <= 0) {
|
|
4185
|
+
this._log('Credit deal watchdog disabled by configuration (TIMING.CREDIT_DEAL_CHECK_INTERVAL_MIN <= 0)');
|
|
4186
|
+
return;
|
|
4187
|
+
}
|
|
4188
|
+
if (this._creditWatchdogInterval) {
|
|
4189
|
+
clearInterval(this._creditWatchdogInterval);
|
|
4190
|
+
this._creditWatchdogInterval = null;
|
|
4191
|
+
}
|
|
4192
|
+
const intervalMs = intervalMin * 60 * TIMING.MILLISECONDS_PER_SECOND;
|
|
4193
|
+
this._creditWatchdogInterval = setInterval(async () => {
|
|
4194
|
+
try {
|
|
4195
|
+
await runtime.runCreditWatchdog();
|
|
4196
|
+
}
|
|
4197
|
+
catch (err) {
|
|
4198
|
+
this._warn(`Credit watchdog error: ${err.message}`);
|
|
4199
|
+
}
|
|
4200
|
+
}, intervalMs);
|
|
4201
|
+
this._log(`Credit deal watchdog started (${intervalMin}min interval)`);
|
|
4202
|
+
}
|
|
4203
|
+
/**
|
|
4204
|
+
* Stop the credit deal watchdog interval.
|
|
4205
|
+
* @returns {void}
|
|
4206
|
+
*/
|
|
4207
|
+
_stopCreditWatchdogInterval() {
|
|
4208
|
+
if (this._creditWatchdogInterval) {
|
|
4209
|
+
clearInterval(this._creditWatchdogInterval);
|
|
4210
|
+
this._creditWatchdogInterval = null;
|
|
4211
|
+
}
|
|
4212
|
+
}
|
|
4213
|
+
async requestGridReset(reason = 'structural change', options = {}) {
|
|
4214
|
+
if (!this.manager || typeof this._performGridResync !== 'function') {
|
|
4215
|
+
return { skipped: true, reason: 'grid resync unavailable' };
|
|
4216
|
+
}
|
|
4217
|
+
const message = reason ? `[CR-RESET] ${reason}` : '[CR-RESET] grid reset requested';
|
|
4218
|
+
// Manual/programmatic resets should advance the persisted center baseline
|
|
4219
|
+
// before rebuilding the grid, unless a caller explicitly disables it.
|
|
4220
|
+
this._log(`${message}; rebuilding grid from fresh on-chain state`, 'info');
|
|
4221
|
+
const resetOptions = {
|
|
4222
|
+
...options,
|
|
4223
|
+
refreshCenterPrice: options.refreshCenterPrice !== false,
|
|
4224
|
+
};
|
|
4225
|
+
if (options.fillLockAlreadyHeld || !this.manager._fillProcessingLock) {
|
|
4226
|
+
return this._performGridResync(resetOptions);
|
|
4227
|
+
}
|
|
4228
|
+
return this.manager._fillProcessingLock.acquire(async () => this._performGridResync(resetOptions));
|
|
4229
|
+
}
|
|
4230
|
+
_wireStructuralGridResyncRequest() {
|
|
4231
|
+
if (!this.manager || this.manager.requestStructuralGridResync)
|
|
4232
|
+
return;
|
|
4233
|
+
this.manager.requestStructuralGridResync = async (reason = 'structural recovery', details = {}) => {
|
|
4234
|
+
if (this._shuttingDown) {
|
|
4235
|
+
return { skipped: true, reason: 'shutting down' };
|
|
4236
|
+
}
|
|
4237
|
+
if (this._structuralGridResyncRunning || this._structuralGridResyncTimer) {
|
|
4238
|
+
return { skipped: true, reason: 'structural grid resync already scheduled' };
|
|
4239
|
+
}
|
|
4240
|
+
const unmatchedCount = Array.isArray(details?.unmatchedChainOrders)
|
|
4241
|
+
? details.unmatchedChainOrders.length
|
|
4242
|
+
: 0;
|
|
4243
|
+
this._structuralGridResyncTimer = setTimeout(async () => {
|
|
4244
|
+
this._structuralGridResyncTimer = null;
|
|
4245
|
+
if (this._shuttingDown)
|
|
4246
|
+
return;
|
|
4247
|
+
this._structuralGridResyncRunning = true;
|
|
4248
|
+
try {
|
|
4249
|
+
const suffix = unmatchedCount > 0 ? ` (${unmatchedCount} unmatched chain order(s))` : '';
|
|
4250
|
+
this._warn(`[RECOVERY] Running structural full grid resync for ${reason}${suffix}`);
|
|
4251
|
+
const resetResult = await this.requestGridReset('rms_structural_grid_resync', {
|
|
4252
|
+
refreshCenterPrice: true,
|
|
4253
|
+
});
|
|
4254
|
+
if (resetResult && this.manager?._recoveryState) {
|
|
4255
|
+
this.manager._recoveryState.attemptCount = 0;
|
|
4256
|
+
this.manager._recoveryState.lastAttemptAt = 0;
|
|
4257
|
+
this.manager._recoveryState.lastFailureAt = 0;
|
|
4258
|
+
}
|
|
4259
|
+
}
|
|
4260
|
+
catch (err) {
|
|
4261
|
+
this._warn(`[RECOVERY] Structural full grid resync failed: ${err.message}`);
|
|
4262
|
+
}
|
|
4263
|
+
finally {
|
|
4264
|
+
this._structuralGridResyncRunning = false;
|
|
4265
|
+
if (this.manager?._recoveryState) {
|
|
4266
|
+
this.manager._recoveryState.structuralResyncRequested = false;
|
|
4267
|
+
}
|
|
4268
|
+
}
|
|
4269
|
+
}, 0);
|
|
4270
|
+
return { scheduled: true };
|
|
4271
|
+
};
|
|
4272
|
+
}
|
|
4273
|
+
/**
|
|
4274
|
+
* Get current metrics for monitoring and debugging.
|
|
4275
|
+
* @returns {Object} Metrics snapshot
|
|
4276
|
+
*/
|
|
4277
|
+
getMetrics() {
|
|
4278
|
+
this.manager?._cleanExpiredLocks?.();
|
|
4279
|
+
return {
|
|
4280
|
+
...this._metrics,
|
|
4281
|
+
queueDepth: this._incomingFillQueue.length,
|
|
4282
|
+
fillProcessingLockActive: this.manager?._fillProcessingLock?.isLocked() || false,
|
|
4283
|
+
divergenceLockActive: this.manager?._divergenceLock?.isLocked() || false,
|
|
4284
|
+
shadowLocksActive: this.manager?.shadowOrderIds?.size || 0,
|
|
4285
|
+
recentFillsTracked: this._recentlyProcessedFills.size
|
|
4286
|
+
};
|
|
4287
|
+
}
|
|
4288
|
+
/**
|
|
4289
|
+
* Execute grid maintenance checks in strict order with pipeline consensus.
|
|
4290
|
+
*
|
|
4291
|
+
* CRITICAL DESIGN: All structural grid modifications are deferred until the pipeline
|
|
4292
|
+
* is empty to prevent "race-to-resize" conditions where the bot attempts to reallocate
|
|
4293
|
+
* temporary fund surpluses from filled orders before their counter-orders/rotations
|
|
4294
|
+
* are placed.
|
|
4295
|
+
*
|
|
4296
|
+
* MAINTENANCE SEQUENCE:
|
|
4297
|
+
* 1. Fund Recalculation (ALWAYS) - Updates internal fund metrics
|
|
4298
|
+
* 2. Pipeline Check (GATE) - Verifies no pending operations
|
|
4299
|
+
* 3. Health Check (IF IDLE) - Detects and cleans dust orders
|
|
4300
|
+
* 4. Divergence Detection (IF IDLE) - Identifies structural mismatches
|
|
4301
|
+
* 5. Grid Resizing (IF IDLE) - Applies size corrections on-chain
|
|
4302
|
+
* 6. Spread Correction (IF IDLE) - Corrects spread after structural work completes
|
|
4303
|
+
*
|
|
4304
|
+
* WHY PIPELINE CONSENSUS MATTERS:
|
|
4305
|
+
* - After a fill, funds temporarily show a "surplus" from the filled order
|
|
4306
|
+
* - If grid maintenance runs immediately, it sees the surplus and triggers a resize
|
|
4307
|
+
* - The resize attempts to allocate funds that will be consumed by pending counter-orders
|
|
4308
|
+
* - This causes cascading trades, fund accounting errors, and grid instability
|
|
4309
|
+
* - Solution: Wait for pipeline to empty (all rotations placed) before resizing
|
|
4310
|
+
*
|
|
4311
|
+
* TIMEOUT SAFETY:
|
|
4312
|
+
* - clearStalePipelineOperations() clears stuck operations after 5-minute timeout
|
|
4313
|
+
* - Called before pipeline check to prevent indefinite blocking
|
|
4314
|
+
* - See manager.clearStalePipelineOperations() for details
|
|
4315
|
+
*
|
|
4316
|
+
* @param {Object} context - Maintenance context for logging.
|
|
4317
|
+
* @private
|
|
4318
|
+
*/
|
|
4319
|
+
async _executeMaintenanceLogic(context) {
|
|
4320
|
+
return DexbotMaintenanceRuntime.executeMaintenanceLogic(this, context);
|
|
4321
|
+
}
|
|
4322
|
+
/**
|
|
4323
|
+
* Cancel dust partial orders immediately — no maps, no timers, no delay.
|
|
4324
|
+
* Each dust order is cancelled on chain and its slot rotated through the
|
|
4325
|
+
* synthetic-fill pipeline.
|
|
4326
|
+
* @param {{ buy: Array, sell: Array, fillLockAlreadyHeld?: boolean }} options
|
|
4327
|
+
* @returns {Promise<{cancelledCount: number, batchResult: {aborted: boolean}|null}>}
|
|
4328
|
+
* @private
|
|
4329
|
+
*/
|
|
4330
|
+
async _cancelDustOrders({ buy: buyDust = [], sell: sellDust = [], fillLockAlreadyHeld = false } = {}) {
|
|
4331
|
+
return DexbotMaintenanceRuntime.cancelDustOrders(this, { buy: buyDust, sell: sellDust, fillLockAlreadyHeld });
|
|
4332
|
+
}
|
|
4333
|
+
/**
|
|
4334
|
+
* Set up the periodic dust health check interval.
|
|
4335
|
+
* Catches partials below the dust threshold that were missed by the post-fill
|
|
4336
|
+
* pipeline (e.g. from a prior bot lifetime after crash/restart).
|
|
4337
|
+
* Cancels dust immediately inside the fill-processing lock.
|
|
4338
|
+
* @private
|
|
4339
|
+
*/
|
|
4340
|
+
_setupDustHealthCheckInterval() {
|
|
4341
|
+
const DUST_HEALTH_CHECK_INTERVAL_MS = 5 * 60 * 1000;
|
|
4342
|
+
this._dustHealthCheckTimer = setInterval(async () => {
|
|
4343
|
+
if (this._shuttingDown || !this.manager)
|
|
4344
|
+
return;
|
|
4345
|
+
try {
|
|
4346
|
+
const health = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan?.bind(this));
|
|
4347
|
+
const allDust = [
|
|
4348
|
+
...(health.buyDustOrders || []),
|
|
4349
|
+
...(health.sellDustOrders || []),
|
|
4350
|
+
];
|
|
4351
|
+
if (allDust.length > 0) {
|
|
4352
|
+
await this.manager._fillProcessingLock.acquire(async () => {
|
|
4353
|
+
await this._cancelDustOrders({
|
|
4354
|
+
buy: health.buyDustOrders,
|
|
4355
|
+
sell: health.sellDustOrders,
|
|
4356
|
+
fillLockAlreadyHeld: true,
|
|
4357
|
+
});
|
|
4358
|
+
});
|
|
4359
|
+
}
|
|
4360
|
+
}
|
|
4361
|
+
catch {
|
|
4362
|
+
// Silently retry next interval
|
|
4363
|
+
}
|
|
4364
|
+
}, DUST_HEALTH_CHECK_INTERVAL_MS);
|
|
4365
|
+
}
|
|
4366
|
+
/**
|
|
4367
|
+
* Perform grid maintenance: fund thresholds, spread condition, grid health, divergence.
|
|
4368
|
+
* Consolidates maintenance checks used during startup, periodic updates, and post-fill.
|
|
4369
|
+
*
|
|
4370
|
+
* ENTRY POINTS:
|
|
4371
|
+
* 1. Startup (line ~681): After grid initialization, ensures grid is healthy
|
|
4372
|
+
* 2. Periodic (line ~2682): Every BLOCKCHAIN_FETCH_INTERVAL_MIN (default 240 min)
|
|
4373
|
+
* 3. Post-Fill (line ~1059): After order fills are rotated
|
|
4374
|
+
*
|
|
4375
|
+
* PIPELINE PROTECTION:
|
|
4376
|
+
* All maintenance operations inside _executeMaintenanceLogic respect isPipelineEmpty().
|
|
4377
|
+
* This prevents grid modifications while fills/rotations/corrections are pending.
|
|
4378
|
+
* See _executeMaintenanceLogic documentation for detailed rationale.
|
|
4379
|
+
*
|
|
4380
|
+
* LOCK ORDERING:
|
|
4381
|
+
* - Canonical order: _fillProcessingLock → _divergenceLock
|
|
4382
|
+
* - This function handles lock acquisition based on fillLockAlreadyHeld parameter
|
|
4383
|
+
* - When called from post-fill context, fill lock is already held
|
|
4384
|
+
* - When called from periodic context, both locks must be acquired
|
|
4385
|
+
* - Matches the order used in _consumeFillQueue to prevent deadlocks
|
|
4386
|
+
*
|
|
4387
|
+
* @param {string} context - Maintenance context for logging (e.g. 'startup', 'periodic', 'post-fill')
|
|
4388
|
+
* @param {Object} options - Maintenance options
|
|
4389
|
+
* @private
|
|
4390
|
+
*/
|
|
4391
|
+
async _runGridMaintenance(context = 'periodic', options = {}) {
|
|
4392
|
+
return DexbotMaintenanceRuntime.runGridMaintenance(this, context, options);
|
|
4393
|
+
}
|
|
4394
|
+
/**
|
|
4395
|
+
* Gracefully shutdown the bot.
|
|
4396
|
+
* Waits for current fill processing to complete, persists state, and stops intervals.
|
|
4397
|
+
* Idempotent: subsequent calls await the first shutdown so duplicate cleanup
|
|
4398
|
+
* invocations do not run the body twice or exit before persistence finishes.
|
|
4399
|
+
* @returns {Promise<void>}
|
|
4400
|
+
*/
|
|
4401
|
+
async shutdown() {
|
|
4402
|
+
if (this._shutdownStarted) {
|
|
4403
|
+
this._log('Shutdown already in progress; ignoring re-entrant call');
|
|
4404
|
+
return this._shutdownPromise || Promise.resolve();
|
|
4405
|
+
}
|
|
4406
|
+
this._shutdownStarted = true;
|
|
4407
|
+
const shutdownImpl = typeof this._shutdownImpl === 'function'
|
|
4408
|
+
? this._shutdownImpl
|
|
4409
|
+
: DEXBot.prototype._shutdownImpl;
|
|
4410
|
+
this._shutdownPromise = shutdownImpl.call(this);
|
|
4411
|
+
return this._shutdownPromise;
|
|
4412
|
+
}
|
|
4413
|
+
async _shutdownImpl() {
|
|
4414
|
+
this._log('Initiating graceful shutdown...');
|
|
4415
|
+
this._shuttingDown = true;
|
|
4416
|
+
this._processedFillStore.setShuttingDown(true);
|
|
4417
|
+
// Stop accepting new work
|
|
4418
|
+
this._stopBlockchainFetchInterval();
|
|
4419
|
+
if (this._triggerDebounceTimer) {
|
|
4420
|
+
clearTimeout(this._triggerDebounceTimer);
|
|
4421
|
+
this._triggerDebounceTimer = null;
|
|
4422
|
+
}
|
|
4423
|
+
if (this._deferredGridResyncTimer) {
|
|
4424
|
+
clearTimeout(this._deferredGridResyncTimer);
|
|
4425
|
+
this._deferredGridResyncTimer = null;
|
|
4426
|
+
}
|
|
4427
|
+
if (this._maintenanceIdleTimer) {
|
|
4428
|
+
clearTimeout(this._maintenanceIdleTimer);
|
|
4429
|
+
this._maintenanceIdleTimer = null;
|
|
4430
|
+
}
|
|
4431
|
+
if (this._credentialRecoveryDeferredTimer) {
|
|
4432
|
+
clearTimeout(this._credentialRecoveryDeferredTimer);
|
|
4433
|
+
this._credentialRecoveryDeferredTimer = null;
|
|
4434
|
+
}
|
|
4435
|
+
if (this._structuralGridResyncTimer) {
|
|
4436
|
+
clearTimeout(this._structuralGridResyncTimer);
|
|
4437
|
+
this._structuralGridResyncTimer = null;
|
|
4438
|
+
}
|
|
4439
|
+
if (this._dustHealthCheckTimer) {
|
|
4440
|
+
clearInterval(this._dustHealthCheckTimer);
|
|
4441
|
+
this._dustHealthCheckTimer = null;
|
|
4442
|
+
}
|
|
4443
|
+
this._stopCreditWatchdogInterval();
|
|
4444
|
+
this._stopCredentialDaemonWatchdogInterval();
|
|
4445
|
+
if (this._creditRuntime) {
|
|
4446
|
+
try {
|
|
4447
|
+
await this._creditRuntime.shutdown();
|
|
4448
|
+
}
|
|
4449
|
+
catch (err) {
|
|
4450
|
+
this._warn(`Failed to persist credit runtime state: ${err.message}`);
|
|
4451
|
+
}
|
|
4452
|
+
}
|
|
4453
|
+
if (this._triggerWatcher && typeof this._triggerWatcher.close === 'function') {
|
|
4454
|
+
try {
|
|
4455
|
+
this._triggerWatcher.close();
|
|
4456
|
+
}
|
|
4457
|
+
catch (err) {
|
|
4458
|
+
this._warn(`Failed to close trigger watcher: ${err.message}`);
|
|
4459
|
+
}
|
|
4460
|
+
finally {
|
|
4461
|
+
this._triggerWatcher = null;
|
|
4462
|
+
}
|
|
4463
|
+
}
|
|
4464
|
+
if (typeof this._fillsUnsubscribe === 'function') {
|
|
4465
|
+
try {
|
|
4466
|
+
await this._fillsUnsubscribe();
|
|
4467
|
+
}
|
|
4468
|
+
catch (err) {
|
|
4469
|
+
this._warn(`Failed to unsubscribe fill listener: ${err.message}`);
|
|
4470
|
+
}
|
|
4471
|
+
finally {
|
|
4472
|
+
this._fillsUnsubscribe = null;
|
|
4473
|
+
}
|
|
4474
|
+
}
|
|
4475
|
+
if (typeof this._reconnectUnregister === 'function') {
|
|
4476
|
+
try {
|
|
4477
|
+
this._reconnectUnregister();
|
|
4478
|
+
}
|
|
4479
|
+
catch (err) {
|
|
4480
|
+
this._warn(`Error unregistering reconnect callback: ${err.message}`);
|
|
4481
|
+
}
|
|
4482
|
+
this._reconnectUnregister = null;
|
|
4483
|
+
}
|
|
4484
|
+
try {
|
|
4485
|
+
await this._stopOpenOrdersSyncLoop();
|
|
4486
|
+
}
|
|
4487
|
+
catch (err) {
|
|
4488
|
+
this._warn(`Error while stopping open-orders sync loop: ${err.message}`);
|
|
4489
|
+
}
|
|
4490
|
+
try {
|
|
4491
|
+
await this._releaseMarketAdapterRuntime('shutdown');
|
|
4492
|
+
}
|
|
4493
|
+
catch (err) {
|
|
4494
|
+
this._warn(`Error while releasing market adapter runtime: ${err.message}`);
|
|
4495
|
+
}
|
|
4496
|
+
// Wait for current fill processing to complete
|
|
4497
|
+
try {
|
|
4498
|
+
if (!this.manager?._fillProcessingLock) {
|
|
4499
|
+
this._warn('Shutdown lock skipped: manager or fillProcessingLock unavailable');
|
|
4500
|
+
}
|
|
4501
|
+
else {
|
|
4502
|
+
const shutdownLockTimeoutMs = this.config?.timing?.SYNC_LOCK_TIMEOUT_MS;
|
|
4503
|
+
let shutdownLockTimer;
|
|
4504
|
+
// AsyncLock starts the callback as soon as the lock is available,
|
|
4505
|
+
// which can be a few ms after the timeout fires. Without this
|
|
4506
|
+
// claim flag, the lock callback and the fallback path would both
|
|
4507
|
+
// run their own _flushProcessedFillPersistence + persistGrid,
|
|
4508
|
+
// racing on the same persistence targets. Set the flag the moment
|
|
4509
|
+
// the timeout fires (and at the top of the lock callback) so
|
|
4510
|
+
// exactly one path performs the flush.
|
|
4511
|
+
let flushClaimed = false;
|
|
4512
|
+
const lockResult = await Promise.race([
|
|
4513
|
+
this.manager._fillProcessingLock.acquire(async () => {
|
|
4514
|
+
if (flushClaimed) {
|
|
4515
|
+
this._log('Shutdown: fallback flush already ran, skipping lock-protected flush');
|
|
4516
|
+
return;
|
|
4517
|
+
}
|
|
4518
|
+
flushClaimed = true;
|
|
4519
|
+
this._log('Fill processing lock acquired for shutdown');
|
|
4520
|
+
// Log any remaining queued fills
|
|
4521
|
+
if (this._incomingFillQueue.length > 0) {
|
|
4522
|
+
this._warn(`${this._incomingFillQueue.length} fills queued but not processed at shutdown`);
|
|
4523
|
+
}
|
|
4524
|
+
await this._flushProcessedFillPersistence('shutdown');
|
|
4525
|
+
// Persist final state
|
|
4526
|
+
if (this.manager && this.accountOrders && this.config?.botKey) {
|
|
4527
|
+
try {
|
|
4528
|
+
await this.manager.persistGrid();
|
|
4529
|
+
this._log('Final grid snapshot persisted');
|
|
4530
|
+
}
|
|
4531
|
+
catch (err) {
|
|
4532
|
+
this._warn(`Failed to persist final state: ${err.message}`);
|
|
4533
|
+
}
|
|
4534
|
+
}
|
|
4535
|
+
}).then(() => 'acquired'),
|
|
4536
|
+
new Promise((resolve) => {
|
|
4537
|
+
shutdownLockTimer = setTimeout(() => {
|
|
4538
|
+
// Claim the flush for the fallback path BEFORE the
|
|
4539
|
+
// race resolves, so if the lock callback starts
|
|
4540
|
+
// microseconds later it sees the claim.
|
|
4541
|
+
flushClaimed = true;
|
|
4542
|
+
resolve('timed-out');
|
|
4543
|
+
}, shutdownLockTimeoutMs);
|
|
4544
|
+
})
|
|
4545
|
+
]).finally(() => {
|
|
4546
|
+
if (shutdownLockTimer)
|
|
4547
|
+
clearTimeout(shutdownLockTimer);
|
|
4548
|
+
});
|
|
4549
|
+
if (lockResult === 'timed-out') {
|
|
4550
|
+
this._warn(`Shutdown: _fillProcessingLock not acquired within ${shutdownLockTimeoutMs}ms — ` +
|
|
4551
|
+
`falling back to best-effort flush without lock.`);
|
|
4552
|
+
// Best-effort flush without the lock. The lock's only
|
|
4553
|
+
// purpose during shutdown is to prevent concurrent
|
|
4554
|
+
// updates; if we're shutting down, no other updater is
|
|
4555
|
+
// running. The risk is if the bot is being restarted
|
|
4556
|
+
// (not stopped) — in which case the same race window
|
|
4557
|
+
// applies. The trade-off is: prefer losing one batch of
|
|
4558
|
+
// recent fills over skipping persistence entirely.
|
|
4559
|
+
try {
|
|
4560
|
+
if (this._incomingFillQueue.length > 0) {
|
|
4561
|
+
this._warn(`${this._incomingFillQueue.length} fills queued at shutdown; ` +
|
|
4562
|
+
`persisting without lock.`);
|
|
4563
|
+
}
|
|
4564
|
+
await this._flushProcessedFillPersistence('shutdown-fallback');
|
|
4565
|
+
if (this.manager && this.accountOrders && this.config?.botKey) {
|
|
4566
|
+
try {
|
|
4567
|
+
await this.manager.persistGrid();
|
|
4568
|
+
this._log('Final grid snapshot persisted (best-effort, lock not held)');
|
|
4569
|
+
}
|
|
4570
|
+
catch (err) {
|
|
4571
|
+
this._warn(`Failed to persist final state (best-effort): ${err.message}`);
|
|
4572
|
+
}
|
|
4573
|
+
}
|
|
4574
|
+
}
|
|
4575
|
+
catch (err) {
|
|
4576
|
+
this._warn(`Best-effort flush during shutdown failed: ${err.message}`);
|
|
4577
|
+
}
|
|
4578
|
+
}
|
|
4579
|
+
// Always reset the fill-consumer watchdog on shutdown completion,
|
|
4580
|
+
// regardless of whether persistGrid ran or succeeded.
|
|
4581
|
+
this._consecutiveConsumeFailures = 0;
|
|
4582
|
+
this._consumeFailureFirstAt = 0;
|
|
4583
|
+
}
|
|
4584
|
+
}
|
|
4585
|
+
catch (err) {
|
|
4586
|
+
this._warn(`Error during shutdown lock acquisition: ${err.message}`);
|
|
4587
|
+
}
|
|
4588
|
+
// Release fund registry allocation
|
|
4589
|
+
if (this.config?.preferredAccount) {
|
|
4590
|
+
const botName = this.config.botKey;
|
|
4591
|
+
if (botName) {
|
|
4592
|
+
fundRegistry.releaseAllocation(this.config.preferredAccount, botName).catch((err) => {
|
|
4593
|
+
this._warn(`Failed to release fund allocation for ${botName}: ${err.message}`);
|
|
4594
|
+
});
|
|
4595
|
+
}
|
|
4596
|
+
}
|
|
4597
|
+
// Log final metrics
|
|
4598
|
+
const metrics = this.getMetrics();
|
|
4599
|
+
this._log(`Shutdown complete. Final metrics: fills=${metrics.fillsProcessed}, batches=${metrics.batchesExecuted}, ` +
|
|
4600
|
+
`avgProcessingTime=${metrics.fillsProcessed > 0 ? Format.formatMetric2(metrics.fillProcessingTimeMs / metrics.fillsProcessed) : 0}ms, ` +
|
|
4601
|
+
`lockContentions=${metrics.lockContentionEvents}, maxQueueDepth=${metrics.maxQueueDepth}`);
|
|
4602
|
+
// Drop botHmacSecret reference from the signing token (V8 string cannot
|
|
4603
|
+
// be zeroed in place, but dropping the reference allows GC to reclaim it).
|
|
4604
|
+
if (this.privateKey && getKeyStore().isDaemonSigningKey(this.privateKey)) {
|
|
4605
|
+
this.privateKey.botHmacSecret = null;
|
|
4606
|
+
}
|
|
4607
|
+
await this.manager?.logger?.flush();
|
|
4608
|
+
}
|
|
4609
|
+
}
|
|
4610
|
+
module.exports = Object.assign(DEXBot, {
|
|
4611
|
+
normalizeBotEntry: require('./bot_settings').normalizeBotEntry
|
|
4612
|
+
});
|
|
4613
|
+
//# sourceMappingURL=dexbot_class.js.map
|