dexbot 1.6.12 → 1.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +2 -2
- package/analysis/README.md +9 -11
- package/analysis/ama_fitting/README.md +5 -3
- package/claw/README.md +1 -0
- package/claw/docs/AI_BOT_LIBRARY_API.md +3 -1
- package/claw/docs/DEXBOT2_TUNING_CHEAT_SHEET.md +4 -4
- package/claw/modules/claw_bridge.ts +0 -3
- package/claw/modules/claw_catalog.ts +8 -4
- package/claw/modules/claw_skill_md.ts +1 -0
- package/claw/modules/dexbot_profiles.ts +59 -15
- package/claw/modules/types.ts +2 -1
- package/claw/package.json +3 -3
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/skills/margin-trading/SKILL.md +1 -1
- package/claw/skills/margin-trading/references/position-management.md +1 -1
- package/claw/tsconfig.json +2 -2
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
- package/dist/analysis/analyze_args.d.ts +12 -0
- package/dist/analysis/analyze_args.js +20 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
- package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
- package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
- package/dist/analysis/bot_fitting/shared_utils.js +124 -8
- package/dist/analysis/chart_ui.d.ts +30 -1
- package/dist/analysis/chart_ui.js +37 -1
- package/dist/analysis/fills_cache.d.ts +3 -23
- package/dist/analysis/fills_cache.js +3 -84
- package/dist/analysis/fills_source.d.ts +31 -1
- package/dist/analysis/fills_source.js +69 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -23
- package/dist/analysis/grid_correction_check.js +15 -58
- package/dist/analysis/math_utils.d.ts +85 -1
- package/dist/analysis/math_utils.js +116 -1
- package/dist/analysis/price_sources.d.ts +22 -1
- package/dist/analysis/price_sources.js +23 -0
- package/dist/analysis/resolve_source.js +31 -10
- package/dist/analysis/trade_profitability.d.ts +6 -0
- package/dist/analysis/trade_profitability.js +23 -97
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
- package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +27 -40
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
- package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
- package/dist/analysis/worker_pool.d.ts +38 -0
- package/dist/analysis/worker_pool.js +67 -0
- package/dist/dexbot.js +67 -48
- package/dist/market_adapter/ama_signal_runner.js +21 -54
- package/dist/market_adapter/candle_utils.d.ts +7 -1
- package/dist/market_adapter/candle_utils.js +11 -1
- package/dist/market_adapter/cli_args.d.ts +22 -0
- package/dist/market_adapter/cli_args.js +30 -0
- package/dist/market_adapter/core/config_normalizers.js +11 -6
- package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
- package/dist/market_adapter/core/kibana_candles.js +3 -2
- package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
- package/dist/market_adapter/core/market_adapter_service.js +93 -158
- package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
- package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
- package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
- package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
- package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
- package/dist/market_adapter/inputs/window_cache.js +52 -177
- package/dist/market_adapter/interval_utils.d.ts +11 -1
- package/dist/market_adapter/interval_utils.js +16 -1
- package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
- package/dist/market_adapter/lp_chart_runner.js +37 -19
- package/dist/market_adapter/market_adapter.js +27 -72
- package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
- package/dist/market_adapter/utils/data_discovery.js +220 -27
- package/dist/market_adapter/utils/month_shards.d.ts +84 -0
- package/dist/market_adapter/utils/month_shards.js +237 -0
- package/dist/modules/account_bots.d.ts +8 -6
- package/dist/modules/account_bots.js +112 -131
- package/dist/modules/account_orders.d.ts +49 -4
- package/dist/modules/account_orders.js +114 -49
- package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
- package/dist/modules/bitshares-native/chain_client.js +14 -13
- package/dist/modules/bitshares-native/crypto/ecc_core.d.ts +1 -2
- package/dist/modules/bitshares-native/crypto/ecc_core.js +1 -1
- package/dist/modules/bitshares-native/resolvers.js +22 -25
- package/dist/modules/bitshares-native/serial/chain_constants.d.ts +1 -2
- package/dist/modules/bitshares-native/serial/chain_constants.js +1 -2
- package/dist/modules/bitshares-native/serial/types.js +22 -50
- package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
- package/dist/modules/bot_defaults.d.ts +18 -3
- package/dist/modules/bot_defaults.js +34 -8
- package/dist/modules/bot_settings.d.ts +5 -1
- package/dist/modules/bot_settings.js +55 -21
- package/dist/modules/broadcast_failure.d.ts +2 -1
- package/dist/modules/chain_orders.d.ts +1 -1
- package/dist/modules/chain_orders.js +51 -96
- package/dist/modules/cli_start_onboarding.d.ts +2 -1
- package/dist/modules/config.js +6 -1
- package/dist/modules/constants.d.ts +14 -6
- package/dist/modules/constants.js +32 -6
- package/dist/modules/cr_planner.d.ts +1 -1
- package/dist/modules/cr_planner.js +1 -5
- package/dist/modules/credit_pricing.d.ts +2 -1
- package/dist/modules/credit_pricing.js +36 -40
- package/dist/modules/credit_runtime.d.ts +7 -0
- package/dist/modules/credit_runtime.js +45 -73
- package/dist/modules/daemon_node_health.d.ts +4 -3
- package/dist/modules/dexbot_class.d.ts +33 -17
- package/dist/modules/dexbot_class.js +113 -61
- package/dist/modules/dexbot_cow_runtime.d.ts +62 -181
- package/dist/modules/dexbot_cow_runtime.js +380 -69
- package/dist/modules/dexbot_credential_client.d.ts +1 -23
- package/dist/modules/dexbot_credential_client.js +2 -3
- package/dist/modules/dexbot_fill_runtime.d.ts +2 -3
- package/dist/modules/dexbot_fill_runtime.js +17 -7
- package/dist/modules/dexbot_maintenance_runtime.d.ts +36 -81
- package/dist/modules/dexbot_maintenance_runtime.js +137 -133
- package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
- package/dist/modules/dexbot_startup_runtime.js +25 -14
- package/dist/modules/dexbot_state_recovery.js +16 -0
- package/dist/modules/general_settings.d.ts +1 -1
- package/dist/modules/graceful_shutdown.js +17 -11
- package/dist/modules/key_store.d.ts +0 -5
- package/dist/modules/key_store.js +1 -2
- package/dist/modules/lazy_shim.d.ts +11 -0
- package/dist/modules/lazy_shim.js +16 -0
- package/dist/modules/node_connect_policy.d.ts +3 -2
- package/dist/modules/node_failure_ledger.d.ts +6 -5
- package/dist/modules/order/accounting.js +11 -11
- package/dist/modules/order/export.js +2 -2
- package/dist/modules/order/format.d.ts +26 -3
- package/dist/modules/order/format.js +33 -3
- package/dist/modules/order/genesis_policy.d.ts +4 -4
- package/dist/modules/order/genesis_policy.js +6 -3
- package/dist/modules/order/grid.d.ts +26 -12
- package/dist/modules/order/grid.js +468 -80
- package/dist/modules/order/grid_reconcile.d.ts +1 -1
- package/dist/modules/order/grid_reconcile.js +5 -5
- package/dist/modules/order/grid_reconcile_internal.js +48 -40
- package/dist/modules/order/index.d.ts +35 -8
- package/dist/modules/order/logger.d.ts +1 -8
- package/dist/modules/order/logger.js +2 -11
- package/dist/modules/order/manager.d.ts +18 -15
- package/dist/modules/order/manager.js +91 -74
- package/dist/modules/order/strategy.d.ts +2 -2
- package/dist/modules/order/strategy.js +51 -21
- package/dist/modules/order/sync_engine.js +4 -0
- package/dist/modules/order/utils/math.d.ts +3 -3
- package/dist/modules/order/utils/math.js +38 -27
- package/dist/modules/order/utils/order.d.ts +58 -58
- package/dist/modules/order/utils/order.js +158 -137
- package/dist/modules/order/utils/system.d.ts +202 -35
- package/dist/modules/order/utils/system.js +409 -31
- package/dist/modules/order/utils/validate.js +28 -6
- package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
- package/dist/modules/paths.js +32 -34
- package/dist/modules/runtime_settings.d.ts +3 -5
- package/dist/modules/runtime_settings.js +3 -3
- package/dist/modules/settings_merge.d.ts +2 -2
- package/dist/modules/socket_json_client.d.ts +2 -1
- package/dist/modules/types.d.ts +89 -30
- package/dist/modules/utils/errors.js +1 -3
- package/dist/modules/version_notice.d.ts +11 -10
- package/dist/pm2.d.ts +0 -1
- package/dist/pm2.js +56 -35
- package/dist/scripts/analyze-git.js +2 -15
- package/dist/scripts/analyze-orders.d.ts +1 -1
- package/dist/scripts/analyze-orders.js +4 -4
- package/dist/scripts/chart_command.js +2 -2
- package/dist/scripts/check_any.d.ts +4 -0
- package/dist/scripts/check_any.js +53 -21
- package/dist/scripts/diagnose-pool-history.js +1 -6
- package/dist/scripts/divergence-calc.js +6 -6
- package/dist/scripts/run-tests.js +0 -1
- package/dist/scripts/sync-version.js +0 -10
- package/dist/scripts/validate_bots.d.ts +1 -1
- package/dist/scripts/validate_bots.js +3 -3
- package/dist/unlock.js +7 -2
- package/docs/BITSHARES_ONBOARDING.md +8 -12
- package/docs/COPY_ON_WRITE_MASTER_PLAN.md +2 -3
- package/docs/COW_INVARIANTS.md +6 -6
- package/docs/DEXBOT_COMPARISON.md +7 -7
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +8 -6
- package/docs/GRID_PRICE_INVARIANT.md +3 -3
- package/docs/GRID_RECALCULATION.md +4 -1
- package/docs/GRID_RECONCILE.md +4 -4
- package/docs/LIFECYCLE.md +11 -11
- package/docs/LOGGING.md +14 -0
- package/docs/MPA_CREDIT_USAGE.md +1 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +12 -4
- package/docs/README.md +3 -3
- package/docs/WORKFLOW.md +1 -0
- package/docs/architecture.md +25 -16
- package/docs/developer_guide.md +7 -2
- package/market_adapter/README.md +6 -4
- package/package.json +6 -9
- package/scripts/README.md +43 -38
- package/tests/README.md +26 -11
- package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
- package/dist/scripts/diagnose-kibana-candles.js +0 -82
- package/scripts/git-viewer.sh +0 -489
- package/scripts/postinstall.js +0 -16
|
@@ -67,8 +67,8 @@ import * as DexbotStartupRuntime from './dexbot_startup_runtime.js';
|
|
|
67
67
|
import cowRuntime from './dexbot_cow_runtime.js';
|
|
68
68
|
import type { ChainOrderLike, CowResult, OpPhase } from './dexbot_cow_runtime.js';
|
|
69
69
|
import { normalizeBotEntry } from './bot_settings.js';
|
|
70
|
-
import type { FillOpLike, FillLike,
|
|
71
|
-
import type { BotLike, AssetPair, IncomingFill,
|
|
70
|
+
import type { FillOpLike, FillLike, ReplaySafeFillAccountingOptions } from './dexbot_fill_runtime.js';
|
|
71
|
+
import type { BotLike, AssetPair, IncomingFill, UnknownRecord, GridConfig, OrderManagerLike, AccountOrdersLike, BotMetrics, ProcessedFillStoreLike, CreditRuntimeLike, FillBatchResult, SigningToken, SpreadCorrectionStall } from './types.js';
|
|
72
72
|
declare class DEXBot implements BotLike {
|
|
73
73
|
static normalizeBotEntry: typeof normalizeBotEntry;
|
|
74
74
|
config: GridConfig;
|
|
@@ -98,6 +98,7 @@ declare class DEXBot implements BotLike {
|
|
|
98
98
|
_credentialRecoveryNeeded: boolean;
|
|
99
99
|
_credentialRecoveryInFlight: boolean;
|
|
100
100
|
_credentialDaemonWatchdogInFlight: boolean;
|
|
101
|
+
_credentialDaemonConsecutiveFailures: number;
|
|
101
102
|
_staleCleanedOrderIds: Map<string, number>;
|
|
102
103
|
_staleCleanupRetentionMs: number;
|
|
103
104
|
_metrics: BotMetrics;
|
|
@@ -174,6 +175,7 @@ declare class DEXBot implements BotLike {
|
|
|
174
175
|
_postRecoveryRebalanceDefers: number;
|
|
175
176
|
_skipEmptyReadConfirmDelay: boolean;
|
|
176
177
|
_spreadFundsExhausted: boolean;
|
|
178
|
+
_spreadCorrectionStall: SpreadCorrectionStall | null;
|
|
177
179
|
_strandedHoldSince: Map<string, number>;
|
|
178
180
|
_structuralGridResyncDeferCount: number;
|
|
179
181
|
_structuralGridResyncDeferStartedAt: number;
|
|
@@ -404,12 +406,7 @@ declare class DEXBot implements BotLike {
|
|
|
404
406
|
* @param {string} [options.persistenceMode='batched']
|
|
405
407
|
* @returns {Promise<any>}
|
|
406
408
|
*/
|
|
407
|
-
_applyReplaySafeTrackedFillAccounting(fill: unknown, fillOp: FillOpLike,
|
|
408
|
-
context?: string;
|
|
409
|
-
logger?: Partial<ManagerLogger>;
|
|
410
|
-
replayMessage?: ReplayMessageFn;
|
|
411
|
-
persistenceMode?: string;
|
|
412
|
-
}): Promise<{
|
|
409
|
+
_applyReplaySafeTrackedFillAccounting(fill: unknown, fillOp: FillOpLike, options?: ReplaySafeFillAccountingOptions): Promise<{
|
|
413
410
|
status: string;
|
|
414
411
|
fillKey: any;
|
|
415
412
|
usedFallbackKey?: undefined;
|
|
@@ -436,12 +433,7 @@ declare class DEXBot implements BotLike {
|
|
|
436
433
|
* @param {string} [options.persistenceMode='batched']
|
|
437
434
|
* @returns {Promise<any>}
|
|
438
435
|
*/
|
|
439
|
-
_applyReplaySafeOrphanFillAccounting(fill: unknown, fillOp: FillOpLike,
|
|
440
|
-
context?: string;
|
|
441
|
-
logger?: Partial<ManagerLogger>;
|
|
442
|
-
replayMessage?: ReplayMessageFn;
|
|
443
|
-
persistenceMode?: string;
|
|
444
|
-
}): Promise<{
|
|
436
|
+
_applyReplaySafeOrphanFillAccounting(fill: unknown, fillOp: FillOpLike, options?: ReplaySafeFillAccountingOptions): Promise<{
|
|
445
437
|
status: string;
|
|
446
438
|
fillKey: any;
|
|
447
439
|
usedFallbackKey?: undefined;
|
|
@@ -803,6 +795,25 @@ declare class DEXBot implements BotLike {
|
|
|
803
795
|
* @returns {void}
|
|
804
796
|
*/
|
|
805
797
|
_setupCredentialDaemonWatchdogInterval(): void;
|
|
798
|
+
/**
|
|
799
|
+
* Consecutive failed watchdog probes required before the credential daemon
|
|
800
|
+
* is declared down. One-liner so the startup log and the probe decision
|
|
801
|
+
* can never disagree about the threshold.
|
|
802
|
+
* @returns {number} Positive integer threshold
|
|
803
|
+
*/
|
|
804
|
+
_credentialDaemonWatchdogFailureThreshold(): number;
|
|
805
|
+
/**
|
|
806
|
+
* Run a single credential-daemon watchdog health probe.
|
|
807
|
+
*
|
|
808
|
+
* A probe failure is only treated as a daemon outage after
|
|
809
|
+
* CREDENTIAL_DAEMON_WATCHDOG_FAILURE_THRESHOLD consecutive failures, so one
|
|
810
|
+
* transient timeout (daemon busy signing) never pauses writes or suspends
|
|
811
|
+
* grid persistence. A confirmed outage sets `_credentialRecoveryNeeded` so
|
|
812
|
+
* the successful-probe path CAN resume persistence — otherwise the
|
|
813
|
+
* suspension lasted until a process restart (production incident).
|
|
814
|
+
* @returns {Promise<void>}
|
|
815
|
+
*/
|
|
816
|
+
_runCredentialDaemonWatchdogProbe(): Promise<void>;
|
|
806
817
|
/**
|
|
807
818
|
* Stop the credential daemon watchdog interval.
|
|
808
819
|
* @returns {void}
|
|
@@ -964,9 +975,9 @@ declare class DEXBot implements BotLike {
|
|
|
964
975
|
* long region (e.g. the structural resync's Phase-2 placement) would
|
|
965
976
|
* starve indefinitely — which also keeps the maintenance idle gate shut
|
|
966
977
|
* forever (the queue-length check returns the full settle delay).
|
|
967
|
-
* Registered via addBroadcastRegionEndListener (fan-out)
|
|
968
|
-
*
|
|
969
|
-
*
|
|
978
|
+
* Registered via addBroadcastRegionEndListener (fan-out) so a second wirer
|
|
979
|
+
* can never silently displace the drain. Re-entry safe: the marker keeps
|
|
980
|
+
* repeated wiring idempotent.
|
|
970
981
|
*/
|
|
971
982
|
_wireBroadcastRegionEndDrain(): void;
|
|
972
983
|
/**
|
|
@@ -974,6 +985,7 @@ declare class DEXBot implements BotLike {
|
|
|
974
985
|
* @returns {Object} Metrics snapshot
|
|
975
986
|
*/
|
|
976
987
|
getMetrics(): {
|
|
988
|
+
boundaryDriftExceedsWindowCount: number;
|
|
977
989
|
queueDepth: number;
|
|
978
990
|
fillProcessingLockActive: boolean;
|
|
979
991
|
divergenceLockActive: boolean;
|
|
@@ -990,6 +1002,10 @@ declare class DEXBot implements BotLike {
|
|
|
990
1002
|
maxQueueDepth: number;
|
|
991
1003
|
gridLockContention: number;
|
|
992
1004
|
fundRecalcCount: number;
|
|
1005
|
+
refusedCommitCount: number;
|
|
1006
|
+
orphanFillCreditedCount: number;
|
|
1007
|
+
restampBuyAboveAnchorCount: number;
|
|
1008
|
+
gapBandPlacementShadowCount: number;
|
|
993
1009
|
};
|
|
994
1010
|
/**
|
|
995
1011
|
* Execute grid maintenance checks in strict order with pipeline consensus.
|
|
@@ -65,6 +65,7 @@
|
|
|
65
65
|
*/
|
|
66
66
|
import { createRequire } from 'node:module';
|
|
67
67
|
const require = createRequire(import.meta.url);
|
|
68
|
+
import { lazyShim } from './lazy_shim.js';
|
|
68
69
|
import * as client from './bitshares_client.js';
|
|
69
70
|
const { BitShares } = client;
|
|
70
71
|
import * as chainKeys from './chain_keys.js';
|
|
@@ -79,16 +80,16 @@ import { PATHS } from './paths.js';
|
|
|
79
80
|
import { path } from './path_api.js';
|
|
80
81
|
import * as Format from './order/format.js';
|
|
81
82
|
import cowRuntime from './dexbot_cow_runtime.js';
|
|
82
|
-
import { ProcessedFillStore
|
|
83
|
+
import { ProcessedFillStore } from './order/processed_fill_store.js';
|
|
83
84
|
import { TIMING, DAEMON_CODES, } from './constants.js';
|
|
84
85
|
import { normalizeBotEntry } from './bot_settings.js';
|
|
85
86
|
import { getErrorMessage, getErrorCode } from './utils/errors.js';
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
87
|
+
const waitForConnected = lazyShim('./bitshares_client', 'waitForConnected');
|
|
88
|
+
const getKeyStore = lazyShim('./key_store', 'getKeyStore');
|
|
89
|
+
const hasExecutableActions = lazyShim('./order/utils/validate', 'hasExecutableActions');
|
|
90
|
+
const getRecalculateTriggerFile = lazyShim('./paths', 'getRecalculateTriggerFile');
|
|
91
|
+
const cloneWeightDistribution = lazyShim('./order/utils/math', 'cloneWeightDistribution');
|
|
92
|
+
const resolveBotRuntimeSettings = lazyShim('./runtime_settings', 'resolveBotRuntimeSettings');
|
|
92
93
|
class DEXBot {
|
|
93
94
|
static normalizeBotEntry;
|
|
94
95
|
config;
|
|
@@ -115,6 +116,7 @@ class DEXBot {
|
|
|
115
116
|
_credentialRecoveryNeeded;
|
|
116
117
|
_credentialRecoveryInFlight;
|
|
117
118
|
_credentialDaemonWatchdogInFlight;
|
|
119
|
+
_credentialDaemonConsecutiveFailures;
|
|
118
120
|
_staleCleanedOrderIds;
|
|
119
121
|
_staleCleanupRetentionMs;
|
|
120
122
|
_metrics;
|
|
@@ -194,6 +196,7 @@ class DEXBot {
|
|
|
194
196
|
_postRecoveryRebalanceDefers;
|
|
195
197
|
_skipEmptyReadConfirmDelay;
|
|
196
198
|
_spreadFundsExhausted;
|
|
199
|
+
_spreadCorrectionStall;
|
|
197
200
|
_strandedHoldSince;
|
|
198
201
|
_structuralGridResyncDeferCount;
|
|
199
202
|
_structuralGridResyncDeferStartedAt;
|
|
@@ -250,6 +253,7 @@ class DEXBot {
|
|
|
250
253
|
this._credentialRecoveryNeeded = false;
|
|
251
254
|
this._credentialRecoveryInFlight = false;
|
|
252
255
|
this._credentialDaemonWatchdogInFlight = false;
|
|
256
|
+
this._credentialDaemonConsecutiveFailures = 0;
|
|
253
257
|
// TTL cache of order IDs freed by stale-order batch cleanup.
|
|
254
258
|
// If an orphan fill arrives for a stale-cleaned order within the retention
|
|
255
259
|
// window, we skip the credit to avoid double-counting freed capital.
|
|
@@ -263,7 +267,12 @@ class DEXBot {
|
|
|
263
267
|
lockContentionEvents: 0,
|
|
264
268
|
gridLockContention: 0,
|
|
265
269
|
fundRecalcCount: 0,
|
|
266
|
-
maxQueueDepth: 0
|
|
270
|
+
maxQueueDepth: 0,
|
|
271
|
+
refusedCommitCount: 0,
|
|
272
|
+
orphanFillCreditedCount: 0,
|
|
273
|
+
restampBuyAboveAnchorCount: 0,
|
|
274
|
+
boundaryDriftExceedsWindowCount: 0,
|
|
275
|
+
gapBandPlacementShadowCount: 0
|
|
267
276
|
};
|
|
268
277
|
// Shutdown state
|
|
269
278
|
this._shuttingDown = false;
|
|
@@ -306,6 +315,11 @@ class DEXBot {
|
|
|
306
315
|
this._postRecoveryRebalanceTimer = null;
|
|
307
316
|
this._deferredFillRetryTimer = null;
|
|
308
317
|
this._lastTargetedDriftSyncAt = 0;
|
|
318
|
+
// Spread-correction stall state: the boolean gate and its companion cause
|
|
319
|
+
// are read by getTargetedSyncReason before the first spread check, so
|
|
320
|
+
// initialize both rather than leaving them undefined.
|
|
321
|
+
this._spreadFundsExhausted = false;
|
|
322
|
+
this._spreadCorrectionStall = null;
|
|
309
323
|
this._lightweightSyncCheckAt = 0;
|
|
310
324
|
this._targetedDriftSyncCooldownMs = this.config.timing?.TARGETED_DRIFT_SYNC_COOLDOWN_MS;
|
|
311
325
|
this._maintenanceCooldownCycles = 0;
|
|
@@ -592,13 +606,8 @@ class DEXBot {
|
|
|
592
606
|
* @param {string} [options.persistenceMode='batched']
|
|
593
607
|
* @returns {Promise<any>}
|
|
594
608
|
*/
|
|
595
|
-
async _applyReplaySafeTrackedFillAccounting(fill, fillOp,
|
|
596
|
-
return DexbotFillRuntime.applyReplaySafeTrackedFillAccounting(this, fill, fillOp,
|
|
597
|
-
context,
|
|
598
|
-
logger,
|
|
599
|
-
replayMessage,
|
|
600
|
-
persistenceMode
|
|
601
|
-
});
|
|
609
|
+
async _applyReplaySafeTrackedFillAccounting(fill, fillOp, options = {}) {
|
|
610
|
+
return DexbotFillRuntime.applyReplaySafeTrackedFillAccounting(this, fill, fillOp, options);
|
|
602
611
|
}
|
|
603
612
|
/**
|
|
604
613
|
* Apply replay-safe fill accounting for orphan fills (grid order not found).
|
|
@@ -611,13 +620,8 @@ class DEXBot {
|
|
|
611
620
|
* @param {string} [options.persistenceMode='batched']
|
|
612
621
|
* @returns {Promise<any>}
|
|
613
622
|
*/
|
|
614
|
-
async _applyReplaySafeOrphanFillAccounting(fill, fillOp,
|
|
615
|
-
return DexbotFillRuntime.applyReplaySafeOrphanFillAccounting(this, fill, fillOp,
|
|
616
|
-
context,
|
|
617
|
-
logger,
|
|
618
|
-
replayMessage,
|
|
619
|
-
persistenceMode
|
|
620
|
-
});
|
|
623
|
+
async _applyReplaySafeOrphanFillAccounting(fill, fillOp, options = {}) {
|
|
624
|
+
return DexbotFillRuntime.applyReplaySafeOrphanFillAccounting(this, fill, fillOp, options);
|
|
621
625
|
}
|
|
622
626
|
/**
|
|
623
627
|
* Refresh dynamic weight distribution from market adapter.
|
|
@@ -1074,8 +1078,8 @@ class DEXBot {
|
|
|
1074
1078
|
}
|
|
1075
1079
|
let anyDeferred = false;
|
|
1076
1080
|
try {
|
|
1077
|
-
const activeSell = this.manager?.config?.
|
|
1078
|
-
const activeBuy = this.manager?.config?.
|
|
1081
|
+
const activeSell = this.manager?.config?.coreOrders?.sell ?? 1;
|
|
1082
|
+
const activeBuy = this.manager?.config?.coreOrders?.buy ?? 1;
|
|
1079
1083
|
const legacyCap = Math.max(Math.floor(activeSell / 2), Math.floor(activeBuy / 2), 1);
|
|
1080
1084
|
const shiftBudget = legacyCap;
|
|
1081
1085
|
this.manager._boundaryShiftBudget = shiftBudget;
|
|
@@ -1304,30 +1308,83 @@ class DEXBot {
|
|
|
1304
1308
|
return;
|
|
1305
1309
|
}
|
|
1306
1310
|
const intervalMs = Math.max(TIMING.DAEMON_PING_TIMEOUT_MS, TIMING.CREDENTIAL_DAEMON_WATCHDOG_MS);
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1311
|
+
this._credentialDaemonWatchdogInterval = setInterval(() => {
|
|
1312
|
+
this._runCredentialDaemonWatchdogProbe().catch((err) => {
|
|
1313
|
+
this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog error: ${getErrorMessage(err)}`, 'warn');
|
|
1314
|
+
});
|
|
1315
|
+
}, intervalMs);
|
|
1316
|
+
if (typeof this._credentialDaemonWatchdogInterval.unref === 'function') {
|
|
1317
|
+
this._credentialDaemonWatchdogInterval.unref();
|
|
1318
|
+
}
|
|
1319
|
+
void this._runCredentialDaemonWatchdogProbe();
|
|
1320
|
+
this._log(`Credential daemon watchdog started (` +
|
|
1321
|
+
`${Math.round(intervalMs / TIMING.MILLISECONDS_PER_SECOND)}s interval, ` +
|
|
1322
|
+
`down after ${this._credentialDaemonWatchdogFailureThreshold()} consecutive probe failures)`);
|
|
1323
|
+
}
|
|
1324
|
+
/**
|
|
1325
|
+
* Consecutive failed watchdog probes required before the credential daemon
|
|
1326
|
+
* is declared down. One-liner so the startup log and the probe decision
|
|
1327
|
+
* can never disagree about the threshold.
|
|
1328
|
+
* @returns {number} Positive integer threshold
|
|
1329
|
+
*/
|
|
1330
|
+
_credentialDaemonWatchdogFailureThreshold() {
|
|
1331
|
+
return Math.max(1, Number(TIMING.CREDENTIAL_DAEMON_WATCHDOG_FAILURE_THRESHOLD) || 2);
|
|
1332
|
+
}
|
|
1333
|
+
/**
|
|
1334
|
+
* Run a single credential-daemon watchdog health probe.
|
|
1335
|
+
*
|
|
1336
|
+
* A probe failure is only treated as a daemon outage after
|
|
1337
|
+
* CREDENTIAL_DAEMON_WATCHDOG_FAILURE_THRESHOLD consecutive failures, so one
|
|
1338
|
+
* transient timeout (daemon busy signing) never pauses writes or suspends
|
|
1339
|
+
* grid persistence. A confirmed outage sets `_credentialRecoveryNeeded` so
|
|
1340
|
+
* the successful-probe path CAN resume persistence — otherwise the
|
|
1341
|
+
* suspension lasted until a process restart (production incident).
|
|
1342
|
+
* @returns {Promise<void>}
|
|
1343
|
+
*/
|
|
1344
|
+
async _runCredentialDaemonWatchdogProbe() {
|
|
1345
|
+
if (this._shuttingDown || !this._isCredentialDaemonWriteRequired())
|
|
1346
|
+
return;
|
|
1347
|
+
// Guard against overlapping ticks: if the previous probe is still
|
|
1348
|
+
// running (slow chain / stall), skip rather than queue a second
|
|
1349
|
+
// pingDaemon and a second recovery attempt.
|
|
1350
|
+
if (this._credentialDaemonWatchdogInFlight)
|
|
1351
|
+
return;
|
|
1352
|
+
this._credentialDaemonWatchdogInFlight = true;
|
|
1353
|
+
const failureThreshold = this._credentialDaemonWatchdogFailureThreshold();
|
|
1354
|
+
try {
|
|
1355
|
+
const token = this.privateKey;
|
|
1316
1356
|
try {
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
this.manager?.logger?.log?.('[CREDENTIAL] Credential daemon responsive again.', 'info');
|
|
1324
|
-
}
|
|
1325
|
-
this._credentialDaemonDown = false;
|
|
1326
|
-
await this._runCredentialRecoveryAfterDaemonRestored();
|
|
1357
|
+
if (getKeyStore().isDaemonSigningKey(token)) {
|
|
1358
|
+
// Use the configured healthcheck timeout (not a hardcoded
|
|
1359
|
+
// tighter value): a daemon busy signing a large batch can
|
|
1360
|
+
// legitimately exceed a 2s probe, and a false "down" here
|
|
1361
|
+
// suspends grid persistence.
|
|
1362
|
+
await chainKeys.pingDaemon(token.accountName, TIMING.DAEMON_PING_TIMEOUT_MS, { socketPath: token.socketPath });
|
|
1327
1363
|
}
|
|
1328
|
-
|
|
1364
|
+
if (this._credentialDaemonDown) {
|
|
1365
|
+
this.manager?.logger?.log?.('[CREDENTIAL] Credential daemon responsive again.', 'info');
|
|
1366
|
+
}
|
|
1367
|
+
this._credentialDaemonDown = false;
|
|
1368
|
+
this._credentialDaemonConsecutiveFailures = 0;
|
|
1369
|
+
await this._runCredentialRecoveryAfterDaemonRestored();
|
|
1370
|
+
}
|
|
1371
|
+
catch (err) {
|
|
1372
|
+
this._credentialDaemonConsecutiveFailures += 1;
|
|
1373
|
+
const errMsg = String(getErrorMessage(err) || '');
|
|
1374
|
+
// Connectivity failures are unambiguous (socket missing/refused):
|
|
1375
|
+
// a hard outage on the first probe. Timeouts may be transient
|
|
1376
|
+
// (daemon busy signing), so they only trip after the
|
|
1377
|
+
// consecutive-failure threshold.
|
|
1378
|
+
const isHardFailure = errMsg.includes('ENOENT') || errMsg.includes('ECONNREFUSED');
|
|
1379
|
+
if (!isHardFailure && this._credentialDaemonConsecutiveFailures < failureThreshold) {
|
|
1380
|
+
// Transient probe failure: tolerate it and re-probe on the
|
|
1381
|
+
// next tick without pausing writes or suspending persistence.
|
|
1382
|
+
this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog probe failed ` +
|
|
1383
|
+
`(${this._credentialDaemonConsecutiveFailures}/${failureThreshold}): ${getErrorMessage(err)}. ` +
|
|
1384
|
+
`Retrying before pausing writes.`, 'warn');
|
|
1385
|
+
}
|
|
1386
|
+
else {
|
|
1329
1387
|
if (!this._credentialDaemonDown) {
|
|
1330
|
-
const errMsg = String(getErrorMessage(err) || '');
|
|
1331
1388
|
let hint = '';
|
|
1332
1389
|
if (errMsg.includes('ENOENT')) {
|
|
1333
1390
|
hint = `Socket file missing at ${token.socketPath}. The credential daemon process may have been killed (e.g. by stray Ctrl-C). Restart it with: dexbot pm2 restart dexbot-cred. If the problem persists, check the daemon log: ${path.join(PATHS.LOGS_DIR, 'dexbot-cred.log')}`;
|
|
@@ -1344,23 +1401,18 @@ class DEXBot {
|
|
|
1344
1401
|
this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog failed: ${getErrorMessage(err)}. ${hint}`, 'error');
|
|
1345
1402
|
}
|
|
1346
1403
|
this._credentialDaemonDown = true;
|
|
1404
|
+
// Mark recovery needed so the successful-probe path
|
|
1405
|
+
// actually runs `_runCredentialRecoveryAfterDaemonRestored`
|
|
1406
|
+
// and RESUME grid persistence. Without this the suspension
|
|
1407
|
+
// below was permanent until restart.
|
|
1408
|
+
this._credentialRecoveryNeeded = true;
|
|
1347
1409
|
this._suspendGridPersistenceForCredentialOutage(`credential daemon watchdog failed: ${getErrorMessage(err)}`);
|
|
1348
1410
|
}
|
|
1349
1411
|
}
|
|
1350
|
-
finally {
|
|
1351
|
-
this._credentialDaemonWatchdogInFlight = false;
|
|
1352
|
-
}
|
|
1353
|
-
};
|
|
1354
|
-
this._credentialDaemonWatchdogInterval = setInterval(() => {
|
|
1355
|
-
probe().catch((err) => {
|
|
1356
|
-
this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog error: ${getErrorMessage(err)}`, 'warn');
|
|
1357
|
-
});
|
|
1358
|
-
}, intervalMs);
|
|
1359
|
-
if (typeof this._credentialDaemonWatchdogInterval.unref === 'function') {
|
|
1360
|
-
this._credentialDaemonWatchdogInterval.unref();
|
|
1361
1412
|
}
|
|
1362
|
-
|
|
1363
|
-
|
|
1413
|
+
finally {
|
|
1414
|
+
this._credentialDaemonWatchdogInFlight = false;
|
|
1415
|
+
}
|
|
1364
1416
|
}
|
|
1365
1417
|
/**
|
|
1366
1418
|
* Stop the credential daemon watchdog interval.
|
|
@@ -1684,9 +1736,9 @@ class DEXBot {
|
|
|
1684
1736
|
* long region (e.g. the structural resync's Phase-2 placement) would
|
|
1685
1737
|
* starve indefinitely — which also keeps the maintenance idle gate shut
|
|
1686
1738
|
* forever (the queue-length check returns the full settle delay).
|
|
1687
|
-
* Registered via addBroadcastRegionEndListener (fan-out)
|
|
1688
|
-
*
|
|
1689
|
-
*
|
|
1739
|
+
* Registered via addBroadcastRegionEndListener (fan-out) so a second wirer
|
|
1740
|
+
* can never silently displace the drain. Re-entry safe: the marker keeps
|
|
1741
|
+
* repeated wiring idempotent.
|
|
1690
1742
|
*/
|
|
1691
1743
|
_wireBroadcastRegionEndDrain() {
|
|
1692
1744
|
const manager = this.manager;
|