dexbot 1.6.12 → 1.6.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/CHANGELOG.md +33 -1
  2. package/analysis/README.md +9 -11
  3. package/analysis/ama_fitting/README.md +5 -3
  4. package/claw/README.md +1 -0
  5. package/claw/docs/AI_BOT_LIBRARY_API.md +2 -0
  6. package/claw/modules/claw_bridge.ts +0 -3
  7. package/claw/modules/claw_skill_md.ts +1 -0
  8. package/claw/package.json +3 -3
  9. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  10. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  11. package/claw/tsconfig.json +2 -2
  12. package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
  13. package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
  14. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
  15. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
  16. package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
  17. package/dist/analysis/analyze_args.d.ts +12 -0
  18. package/dist/analysis/analyze_args.js +20 -0
  19. package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
  20. package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
  21. package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
  22. package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
  23. package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
  24. package/dist/analysis/bot_fitting/shared_utils.js +124 -8
  25. package/dist/analysis/chart_ui.d.ts +30 -1
  26. package/dist/analysis/chart_ui.js +37 -1
  27. package/dist/analysis/fills_cache.d.ts +3 -23
  28. package/dist/analysis/fills_cache.js +3 -84
  29. package/dist/analysis/fills_source.d.ts +31 -1
  30. package/dist/analysis/fills_source.js +69 -1
  31. package/dist/analysis/grid_correction_check.d.ts +1 -23
  32. package/dist/analysis/grid_correction_check.js +15 -58
  33. package/dist/analysis/math_utils.d.ts +85 -1
  34. package/dist/analysis/math_utils.js +116 -1
  35. package/dist/analysis/price_sources.d.ts +22 -1
  36. package/dist/analysis/price_sources.js +23 -0
  37. package/dist/analysis/resolve_source.js +31 -10
  38. package/dist/analysis/trade_profitability.d.ts +6 -0
  39. package/dist/analysis/trade_profitability.js +23 -97
  40. package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
  41. package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
  42. package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +2 -15
  43. package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
  44. package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
  45. package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
  46. package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
  47. package/dist/analysis/worker_pool.d.ts +38 -0
  48. package/dist/analysis/worker_pool.js +67 -0
  49. package/dist/dexbot.js +67 -48
  50. package/dist/market_adapter/ama_signal_runner.js +21 -54
  51. package/dist/market_adapter/candle_utils.d.ts +7 -1
  52. package/dist/market_adapter/candle_utils.js +11 -1
  53. package/dist/market_adapter/cli_args.d.ts +22 -0
  54. package/dist/market_adapter/cli_args.js +30 -0
  55. package/dist/market_adapter/core/config_normalizers.js +11 -6
  56. package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
  57. package/dist/market_adapter/core/kibana_candles.js +3 -2
  58. package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
  59. package/dist/market_adapter/core/market_adapter_service.js +93 -158
  60. package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
  61. package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
  62. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
  63. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
  64. package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
  65. package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
  66. package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
  67. package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
  68. package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
  69. package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
  70. package/dist/market_adapter/inputs/window_cache.js +52 -177
  71. package/dist/market_adapter/interval_utils.d.ts +11 -1
  72. package/dist/market_adapter/interval_utils.js +16 -1
  73. package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
  74. package/dist/market_adapter/lp_chart_runner.js +37 -19
  75. package/dist/market_adapter/market_adapter.js +27 -72
  76. package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
  77. package/dist/market_adapter/utils/data_discovery.js +220 -27
  78. package/dist/market_adapter/utils/month_shards.d.ts +84 -0
  79. package/dist/market_adapter/utils/month_shards.js +237 -0
  80. package/dist/modules/account_bots.d.ts +1 -0
  81. package/dist/modules/account_bots.js +102 -121
  82. package/dist/modules/account_orders.d.ts +13 -0
  83. package/dist/modules/account_orders.js +38 -44
  84. package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
  85. package/dist/modules/bitshares-native/chain_client.js +14 -13
  86. package/dist/modules/bitshares-native/resolvers.js +22 -25
  87. package/dist/modules/bitshares-native/serial/types.js +22 -50
  88. package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
  89. package/dist/modules/bot_defaults.d.ts +2 -1
  90. package/dist/modules/bot_settings.d.ts +1 -1
  91. package/dist/modules/broadcast_failure.d.ts +2 -1
  92. package/dist/modules/chain_orders.d.ts +1 -1
  93. package/dist/modules/chain_orders.js +51 -96
  94. package/dist/modules/cli_start_onboarding.d.ts +2 -1
  95. package/dist/modules/constants.d.ts +7 -1
  96. package/dist/modules/constants.js +20 -1
  97. package/dist/modules/cr_planner.d.ts +1 -1
  98. package/dist/modules/cr_planner.js +1 -5
  99. package/dist/modules/credit_pricing.d.ts +2 -1
  100. package/dist/modules/credit_pricing.js +36 -40
  101. package/dist/modules/credit_runtime.d.ts +7 -0
  102. package/dist/modules/credit_runtime.js +45 -73
  103. package/dist/modules/daemon_node_health.d.ts +4 -3
  104. package/dist/modules/dexbot_class.d.ts +28 -17
  105. package/dist/modules/dexbot_class.js +105 -58
  106. package/dist/modules/dexbot_cow_runtime.d.ts +0 -7
  107. package/dist/modules/dexbot_cow_runtime.js +54 -28
  108. package/dist/modules/dexbot_credential_client.d.ts +0 -7
  109. package/dist/modules/dexbot_credential_client.js +1 -2
  110. package/dist/modules/dexbot_fill_runtime.d.ts +1 -2
  111. package/dist/modules/dexbot_fill_runtime.js +5 -6
  112. package/dist/modules/dexbot_maintenance_runtime.d.ts +20 -3
  113. package/dist/modules/dexbot_maintenance_runtime.js +84 -69
  114. package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
  115. package/dist/modules/dexbot_startup_runtime.js +13 -14
  116. package/dist/modules/general_settings.d.ts +1 -1
  117. package/dist/modules/graceful_shutdown.js +17 -11
  118. package/dist/modules/key_store.js +0 -1
  119. package/dist/modules/lazy_shim.d.ts +11 -0
  120. package/dist/modules/lazy_shim.js +16 -0
  121. package/dist/modules/node_connect_policy.d.ts +3 -2
  122. package/dist/modules/node_failure_ledger.d.ts +6 -5
  123. package/dist/modules/order/format.d.ts +12 -3
  124. package/dist/modules/order/format.js +15 -3
  125. package/dist/modules/order/genesis_policy.d.ts +3 -3
  126. package/dist/modules/order/genesis_policy.js +5 -2
  127. package/dist/modules/order/grid.d.ts +4 -2
  128. package/dist/modules/order/grid.js +91 -35
  129. package/dist/modules/order/index.d.ts +1 -0
  130. package/dist/modules/order/logger.d.ts +1 -8
  131. package/dist/modules/order/logger.js +0 -9
  132. package/dist/modules/order/manager.d.ts +9 -13
  133. package/dist/modules/order/manager.js +33 -50
  134. package/dist/modules/order/utils/order.js +39 -35
  135. package/dist/modules/order/utils/system.d.ts +9 -29
  136. package/dist/modules/order/utils/system.js +17 -2
  137. package/dist/modules/order/utils/validate.js +2 -2
  138. package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
  139. package/dist/modules/paths.js +32 -34
  140. package/dist/modules/runtime_settings.d.ts +2 -4
  141. package/dist/modules/settings_merge.d.ts +2 -2
  142. package/dist/modules/socket_json_client.d.ts +2 -1
  143. package/dist/modules/types.d.ts +42 -27
  144. package/dist/modules/utils/errors.js +1 -3
  145. package/dist/modules/version_notice.d.ts +11 -10
  146. package/dist/pm2.d.ts +0 -1
  147. package/dist/pm2.js +56 -35
  148. package/dist/scripts/analyze-git.js +2 -15
  149. package/dist/scripts/chart_command.js +2 -2
  150. package/dist/scripts/check_any.d.ts +4 -0
  151. package/dist/scripts/check_any.js +53 -21
  152. package/dist/scripts/diagnose-pool-history.js +1 -6
  153. package/dist/scripts/run-tests.js +0 -1
  154. package/dist/scripts/sync-version.js +0 -10
  155. package/docs/BITSHARES_ONBOARDING.md +8 -12
  156. package/docs/COPY_ON_WRITE_MASTER_PLAN.md +1 -1
  157. package/docs/DEXBOT_COMPARISON.md +6 -6
  158. package/docs/EVOLUTION.md +9 -8
  159. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  160. package/docs/README.md +1 -1
  161. package/docs/WORKFLOW.md +1 -0
  162. package/docs/architecture.md +19 -10
  163. package/market_adapter/README.md +6 -4
  164. package/package.json +6 -9
  165. package/scripts/README.md +43 -38
  166. package/tests/README.md +25 -10
  167. package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
  168. package/dist/scripts/diagnose-kibana-candles.js +0 -82
  169. package/scripts/git-viewer.sh +0 -489
  170. package/scripts/postinstall.js +0 -16
@@ -65,6 +65,7 @@
65
65
  */
66
66
  import { createRequire } from 'node:module';
67
67
  const require = createRequire(import.meta.url);
68
+ import { lazyShim } from './lazy_shim.js';
68
69
  import * as client from './bitshares_client.js';
69
70
  const { BitShares } = client;
70
71
  import * as chainKeys from './chain_keys.js';
@@ -79,16 +80,16 @@ import { PATHS } from './paths.js';
79
80
  import { path } from './path_api.js';
80
81
  import * as Format from './order/format.js';
81
82
  import cowRuntime from './dexbot_cow_runtime.js';
82
- import { ProcessedFillStore, PROCESSED_FILL_PERSISTENCE_MODES } from './order/processed_fill_store.js';
83
+ import { ProcessedFillStore } from './order/processed_fill_store.js';
83
84
  import { TIMING, DAEMON_CODES, } from './constants.js';
84
85
  import { normalizeBotEntry } from './bot_settings.js';
85
86
  import { getErrorMessage, getErrorCode } from './utils/errors.js';
86
- function waitForConnected(...args) { return require('./bitshares_client').waitForConnected(...args); }
87
- function getKeyStore(...args) { return require('./key_store').getKeyStore(...args); }
88
- function hasExecutableActions(...args) { return require('./order/utils/validate').hasExecutableActions(...args); }
89
- function getRecalculateTriggerFile(...args) { return require('./paths').getRecalculateTriggerFile(...args); }
90
- function cloneWeightDistribution(...args) { return require('./order/utils/math').cloneWeightDistribution(...args); }
91
- function resolveBotRuntimeSettings(...args) { return require('./runtime_settings').resolveBotRuntimeSettings(...args); }
87
+ const waitForConnected = lazyShim('./bitshares_client', 'waitForConnected');
88
+ const getKeyStore = lazyShim('./key_store', 'getKeyStore');
89
+ const hasExecutableActions = lazyShim('./order/utils/validate', 'hasExecutableActions');
90
+ const getRecalculateTriggerFile = lazyShim('./paths', 'getRecalculateTriggerFile');
91
+ const cloneWeightDistribution = lazyShim('./order/utils/math', 'cloneWeightDistribution');
92
+ const resolveBotRuntimeSettings = lazyShim('./runtime_settings', 'resolveBotRuntimeSettings');
92
93
  class DEXBot {
93
94
  static normalizeBotEntry;
94
95
  config;
@@ -115,6 +116,7 @@ class DEXBot {
115
116
  _credentialRecoveryNeeded;
116
117
  _credentialRecoveryInFlight;
117
118
  _credentialDaemonWatchdogInFlight;
119
+ _credentialDaemonConsecutiveFailures;
118
120
  _staleCleanedOrderIds;
119
121
  _staleCleanupRetentionMs;
120
122
  _metrics;
@@ -194,6 +196,7 @@ class DEXBot {
194
196
  _postRecoveryRebalanceDefers;
195
197
  _skipEmptyReadConfirmDelay;
196
198
  _spreadFundsExhausted;
199
+ _spreadCorrectionStall;
197
200
  _strandedHoldSince;
198
201
  _structuralGridResyncDeferCount;
199
202
  _structuralGridResyncDeferStartedAt;
@@ -250,6 +253,7 @@ class DEXBot {
250
253
  this._credentialRecoveryNeeded = false;
251
254
  this._credentialRecoveryInFlight = false;
252
255
  this._credentialDaemonWatchdogInFlight = false;
256
+ this._credentialDaemonConsecutiveFailures = 0;
253
257
  // TTL cache of order IDs freed by stale-order batch cleanup.
254
258
  // If an orphan fill arrives for a stale-cleaned order within the retention
255
259
  // window, we skip the credit to avoid double-counting freed capital.
@@ -306,6 +310,11 @@ class DEXBot {
306
310
  this._postRecoveryRebalanceTimer = null;
307
311
  this._deferredFillRetryTimer = null;
308
312
  this._lastTargetedDriftSyncAt = 0;
313
+ // Spread-correction stall state: the boolean gate and its companion cause
314
+ // are read by getTargetedSyncReason before the first spread check, so
315
+ // initialize both rather than leaving them undefined.
316
+ this._spreadFundsExhausted = false;
317
+ this._spreadCorrectionStall = null;
309
318
  this._lightweightSyncCheckAt = 0;
310
319
  this._targetedDriftSyncCooldownMs = this.config.timing?.TARGETED_DRIFT_SYNC_COOLDOWN_MS;
311
320
  this._maintenanceCooldownCycles = 0;
@@ -592,13 +601,8 @@ class DEXBot {
592
601
  * @param {string} [options.persistenceMode='batched']
593
602
  * @returns {Promise<any>}
594
603
  */
595
- async _applyReplaySafeTrackedFillAccounting(fill, fillOp, { context, logger = this.manager?.logger, replayMessage, persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.BATCHED } = {}) {
596
- return DexbotFillRuntime.applyReplaySafeTrackedFillAccounting(this, fill, fillOp, {
597
- context,
598
- logger,
599
- replayMessage,
600
- persistenceMode
601
- });
604
+ async _applyReplaySafeTrackedFillAccounting(fill, fillOp, options = {}) {
605
+ return DexbotFillRuntime.applyReplaySafeTrackedFillAccounting(this, fill, fillOp, options);
602
606
  }
603
607
  /**
604
608
  * Apply replay-safe fill accounting for orphan fills (grid order not found).
@@ -611,13 +615,8 @@ class DEXBot {
611
615
  * @param {string} [options.persistenceMode='batched']
612
616
  * @returns {Promise<any>}
613
617
  */
614
- async _applyReplaySafeOrphanFillAccounting(fill, fillOp, { context, logger = this.manager?.logger, replayMessage, persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.BATCHED } = {}) {
615
- return DexbotFillRuntime.applyReplaySafeOrphanFillAccounting(this, fill, fillOp, {
616
- context,
617
- logger,
618
- replayMessage,
619
- persistenceMode
620
- });
618
+ async _applyReplaySafeOrphanFillAccounting(fill, fillOp, options = {}) {
619
+ return DexbotFillRuntime.applyReplaySafeOrphanFillAccounting(this, fill, fillOp, options);
621
620
  }
622
621
  /**
623
622
  * Refresh dynamic weight distribution from market adapter.
@@ -1304,30 +1303,83 @@ class DEXBot {
1304
1303
  return;
1305
1304
  }
1306
1305
  const intervalMs = Math.max(TIMING.DAEMON_PING_TIMEOUT_MS, TIMING.CREDENTIAL_DAEMON_WATCHDOG_MS);
1307
- const probe = async () => {
1308
- if (this._shuttingDown || !this._isCredentialDaemonWriteRequired())
1309
- return;
1310
- // Guard against overlapping ticks: if the previous probe is still
1311
- // running (slow chain / stall), skip rather than queue a second
1312
- // pingDaemon and a second recovery attempt.
1313
- if (this._credentialDaemonWatchdogInFlight)
1314
- return;
1315
- this._credentialDaemonWatchdogInFlight = true;
1306
+ this._credentialDaemonWatchdogInterval = setInterval(() => {
1307
+ this._runCredentialDaemonWatchdogProbe().catch((err) => {
1308
+ this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog error: ${getErrorMessage(err)}`, 'warn');
1309
+ });
1310
+ }, intervalMs);
1311
+ if (typeof this._credentialDaemonWatchdogInterval.unref === 'function') {
1312
+ this._credentialDaemonWatchdogInterval.unref();
1313
+ }
1314
+ void this._runCredentialDaemonWatchdogProbe();
1315
+ this._log(`Credential daemon watchdog started (` +
1316
+ `${Math.round(intervalMs / TIMING.MILLISECONDS_PER_SECOND)}s interval, ` +
1317
+ `down after ${this._credentialDaemonWatchdogFailureThreshold()} consecutive probe failures)`);
1318
+ }
1319
+ /**
1320
+ * Consecutive failed watchdog probes required before the credential daemon
1321
+ * is declared down. One-liner so the startup log and the probe decision
1322
+ * can never disagree about the threshold.
1323
+ * @returns {number} Positive integer threshold
1324
+ */
1325
+ _credentialDaemonWatchdogFailureThreshold() {
1326
+ return Math.max(1, Number(TIMING.CREDENTIAL_DAEMON_WATCHDOG_FAILURE_THRESHOLD) || 2);
1327
+ }
1328
+ /**
1329
+ * Run a single credential-daemon watchdog health probe.
1330
+ *
1331
+ * A probe failure is only treated as a daemon outage after
1332
+ * CREDENTIAL_DAEMON_WATCHDOG_FAILURE_THRESHOLD consecutive failures, so one
1333
+ * transient timeout (daemon busy signing) never pauses writes or suspends
1334
+ * grid persistence. A confirmed outage sets `_credentialRecoveryNeeded` so
1335
+ * the successful-probe path CAN resume persistence — otherwise the
1336
+ * suspension lasted until a process restart (production incident).
1337
+ * @returns {Promise<void>}
1338
+ */
1339
+ async _runCredentialDaemonWatchdogProbe() {
1340
+ if (this._shuttingDown || !this._isCredentialDaemonWriteRequired())
1341
+ return;
1342
+ // Guard against overlapping ticks: if the previous probe is still
1343
+ // running (slow chain / stall), skip rather than queue a second
1344
+ // pingDaemon and a second recovery attempt.
1345
+ if (this._credentialDaemonWatchdogInFlight)
1346
+ return;
1347
+ this._credentialDaemonWatchdogInFlight = true;
1348
+ const failureThreshold = this._credentialDaemonWatchdogFailureThreshold();
1349
+ try {
1350
+ const token = this.privateKey;
1316
1351
  try {
1317
- const token = this.privateKey;
1318
- try {
1319
- if (getKeyStore().isDaemonSigningKey(token)) {
1320
- await chainKeys.pingDaemon(token.accountName, 2000, { socketPath: token.socketPath });
1321
- }
1322
- if (this._credentialDaemonDown) {
1323
- this.manager?.logger?.log?.('[CREDENTIAL] Credential daemon responsive again.', 'info');
1324
- }
1325
- this._credentialDaemonDown = false;
1326
- await this._runCredentialRecoveryAfterDaemonRestored();
1352
+ if (getKeyStore().isDaemonSigningKey(token)) {
1353
+ // Use the configured healthcheck timeout (not a hardcoded
1354
+ // tighter value): a daemon busy signing a large batch can
1355
+ // legitimately exceed a 2s probe, and a false "down" here
1356
+ // suspends grid persistence.
1357
+ await chainKeys.pingDaemon(token.accountName, TIMING.DAEMON_PING_TIMEOUT_MS, { socketPath: token.socketPath });
1327
1358
  }
1328
- catch (err) {
1359
+ if (this._credentialDaemonDown) {
1360
+ this.manager?.logger?.log?.('[CREDENTIAL] Credential daemon responsive again.', 'info');
1361
+ }
1362
+ this._credentialDaemonDown = false;
1363
+ this._credentialDaemonConsecutiveFailures = 0;
1364
+ await this._runCredentialRecoveryAfterDaemonRestored();
1365
+ }
1366
+ catch (err) {
1367
+ this._credentialDaemonConsecutiveFailures += 1;
1368
+ const errMsg = String(getErrorMessage(err) || '');
1369
+ // Connectivity failures are unambiguous (socket missing/refused):
1370
+ // a hard outage on the first probe. Timeouts may be transient
1371
+ // (daemon busy signing), so they only trip after the
1372
+ // consecutive-failure threshold.
1373
+ const isHardFailure = errMsg.includes('ENOENT') || errMsg.includes('ECONNREFUSED');
1374
+ if (!isHardFailure && this._credentialDaemonConsecutiveFailures < failureThreshold) {
1375
+ // Transient probe failure: tolerate it and re-probe on the
1376
+ // next tick without pausing writes or suspending persistence.
1377
+ this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog probe failed ` +
1378
+ `(${this._credentialDaemonConsecutiveFailures}/${failureThreshold}): ${getErrorMessage(err)}. ` +
1379
+ `Retrying before pausing writes.`, 'warn');
1380
+ }
1381
+ else {
1329
1382
  if (!this._credentialDaemonDown) {
1330
- const errMsg = String(getErrorMessage(err) || '');
1331
1383
  let hint = '';
1332
1384
  if (errMsg.includes('ENOENT')) {
1333
1385
  hint = `Socket file missing at ${token.socketPath}. The credential daemon process may have been killed (e.g. by stray Ctrl-C). Restart it with: dexbot pm2 restart dexbot-cred. If the problem persists, check the daemon log: ${path.join(PATHS.LOGS_DIR, 'dexbot-cred.log')}`;
@@ -1344,23 +1396,18 @@ class DEXBot {
1344
1396
  this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog failed: ${getErrorMessage(err)}. ${hint}`, 'error');
1345
1397
  }
1346
1398
  this._credentialDaemonDown = true;
1399
+ // Mark recovery needed so the successful-probe path
1400
+ // actually runs `_runCredentialRecoveryAfterDaemonRestored`
1401
+ // and RESUME grid persistence. Without this the suspension
1402
+ // below was permanent until restart.
1403
+ this._credentialRecoveryNeeded = true;
1347
1404
  this._suspendGridPersistenceForCredentialOutage(`credential daemon watchdog failed: ${getErrorMessage(err)}`);
1348
1405
  }
1349
1406
  }
1350
- finally {
1351
- this._credentialDaemonWatchdogInFlight = false;
1352
- }
1353
- };
1354
- this._credentialDaemonWatchdogInterval = setInterval(() => {
1355
- probe().catch((err) => {
1356
- this.manager?.logger?.log?.(`[CREDENTIAL] Credential daemon watchdog error: ${getErrorMessage(err)}`, 'warn');
1357
- });
1358
- }, intervalMs);
1359
- if (typeof this._credentialDaemonWatchdogInterval.unref === 'function') {
1360
- this._credentialDaemonWatchdogInterval.unref();
1361
1407
  }
1362
- void probe();
1363
- this._log(`Credential daemon watchdog started (${Math.round(intervalMs / TIMING.MILLISECONDS_PER_SECOND)}s interval)`);
1408
+ finally {
1409
+ this._credentialDaemonWatchdogInFlight = false;
1410
+ }
1364
1411
  }
1365
1412
  /**
1366
1413
  * Stop the credential daemon watchdog interval.
@@ -1684,9 +1731,9 @@ class DEXBot {
1684
1731
  * long region (e.g. the structural resync's Phase-2 placement) would
1685
1732
  * starve indefinitely — which also keeps the maintenance idle gate shut
1686
1733
  * forever (the queue-length check returns the full settle delay).
1687
- * Registered via addBroadcastRegionEndListener (fan-out) rather than the
1688
- * legacy single slot, so a second wirer can never silently displace the
1689
- * drain. Re-entry safe: the marker keeps repeated wiring idempotent.
1734
+ * Registered via addBroadcastRegionEndListener (fan-out) so a second wirer
1735
+ * can never silently displace the drain. Re-entry safe: the marker keeps
1736
+ * repeated wiring idempotent.
1690
1737
  */
1691
1738
  _wireBroadcastRegionEndDrain() {
1692
1739
  const manager = this.manager;
@@ -58,13 +58,6 @@ export interface CowResult extends JsonObj {
58
58
  refillSlotIds?: string[];
59
59
  fills?: unknown[];
60
60
  }
61
- /** Result of the pre-broadcast guard chain. */
62
- export interface PreBroadcastGuardResult extends JsonObj {
63
- proceed: boolean;
64
- result?: JsonObj;
65
- crossingCandidates?: unknown[];
66
- intraBatchCandidates?: unknown[];
67
- }
68
61
  /** A CREATE op context with its resolved order + integer tuple. */
69
62
  type CreateContext = OpPhase & {
70
63
  finalInts: FinalInts;
@@ -13,7 +13,7 @@ const { readOpenOrdersWithMetaSafe } = chainOrdersModule;
13
13
  import { BroadcastUncertainError as BroadcastUncertainErrorBinding } from './dexbot_credential_client.js';
14
14
  const BroadcastUncertainError = BroadcastUncertainErrorBinding;
15
15
  import * as orderUtils from './order/utils/order.js';
16
- import { sleep, setLastFillPivot } from './order/utils/system.js';
16
+ import { sleep, setLastFillPivot, fireStructuralResync } from './order/utils/system.js';
17
17
  const { buildCreateOrderArgs, buildCreateOpFingerprint, extractBatchOperationResults, formatUnmatchedChainOrder, convertToSpreadPlaceholder, toRailHolePlaceholder, buildOutsideInPairGroups, isOrderPlaced, chainOrderUnchangedFromCache, detectCrossedBookPlan, collectKnownOnChainOrderIds, } = orderUtils;
18
18
  import * as validate from './order/utils/validate.js';
19
19
  const { validateCreateTargetSlots, evaluateCommit, hasExecutableActions, stampGapEvacuationRotation } = validate;
@@ -70,7 +70,7 @@ function hasSlotPriceCollision(items, targetPrice, precision, excludeId, predica
70
70
  return null;
71
71
  }
72
72
  import * as constantsModule from './constants.js';
73
- const { COW_ACTIONS, COW_PERFORMANCE, ORDER_STATES, ORDER_TYPES, REBALANCE_STATES, } = constantsModule;
73
+ const { COW_ACTIONS, COW_ORIGINS, COW_PERFORMANCE, ORDER_STATES, ORDER_TYPES, REBALANCE_STATES, } = constantsModule;
74
74
  import { acquireIfNotHeld } from './order/async_lock.js';
75
75
  import * as FormatModule from './order/format.js';
76
76
  const Format = FormatModule;
@@ -912,11 +912,22 @@ async function resyncIfUnreconciled(bot, err, pending, chainSnapshot, adopted, a
912
912
  // chain snapshot.
913
913
  const alreadyScheduled = bot._structuralGridResyncRunning || bot._structuralGridResyncTimer;
914
914
  if (!alreadyScheduled && chainSnapshot.length > 0) {
915
+ // "Unreconciled" must mean chain orders absent from MASTER, not merely
916
+ // absent from this batch's adopted set. Every resting order master
917
+ // already tracks is reconciled by definition; counting those as
918
+ // unreconciled forced a structural resync (grid rebuild + LAST-FILL
919
+ // pivot clear) after every uncertain broadcast on a populated grid.
915
920
  const reconciledOrderIds = new Set(adopted.map((a) => a.match?.id).filter(Boolean));
921
+ try {
922
+ const known = collectKnownOnChainOrderIds(bot.manager, null, null).masterIds;
923
+ for (const id of known)
924
+ reconciledOrderIds.add(id);
925
+ }
926
+ catch { /* best-effort: fall back to the adopted-only set */ }
916
927
  const unreconciledCount = chainSnapshot.filter((o) => !reconciledOrderIds.has(o.id)).length;
917
928
  if (unreconciledCount > 0) {
918
- bot.manager.logger.log(`[COW][UNCERTAIN] ${unreconciledCount} chain order(s) remain unreconciled after uncertain broadcast recovery. ` +
919
- `These may be legitimate pre-existing orders or leftovers from a prior cycle. Requesting structural resync.`, 'warn');
929
+ bot.manager.logger.log(`[COW][UNCERTAIN] ${unreconciledCount} chain order(s) unknown to master remain after uncertain ` +
930
+ `broadcast recovery (neither adopted nor tracked). Requesting structural resync.`, 'warn');
920
931
  await requestStructuralResync(bot, 'unreconciled orders after uncertain broadcast', {
921
932
  batchId: err?.batchId || null,
922
933
  pendingCount: pending.length,
@@ -1714,21 +1725,13 @@ function considerGridPriceInvariantEscalation(bot, slotId, price, inv, site) {
1714
1725
  `${Number.isFinite(actual) ? Format.formatPrice6(actual) : 'n/a'} ` +
1715
1726
  `(genesis ${Number.isFinite(expected) ? Format.formatPrice6(expected) : 'n/a'}); ` +
1716
1727
  `requesting structural resync to repair the slot`, 'error');
1717
- try {
1718
- const res = bot.manager.requestStructuralGridResync('grid-price-invariant-violation', {
1719
- slotId: String(slotId),
1720
- expected: Number.isFinite(expected) ? expected : null,
1721
- actual: Number.isFinite(actual) ? actual : null,
1722
- site,
1723
- streak,
1724
- });
1725
- res?.catch?.((err) => {
1726
- bot.manager?.logger?.log?.(`[GRID-PRICE-INVARIANT] Structural resync request failed: ${getErrorMessage(err)}`, 'error');
1727
- });
1728
- }
1729
- catch (err) {
1730
- bot.manager?.logger?.log?.(`[GRID-PRICE-INVARIANT] Structural resync request failed: ${getErrorMessage(err)}`, 'error');
1731
- }
1728
+ fireStructuralResync(bot, 'grid-price-invariant-violation', {
1729
+ slotId: String(slotId),
1730
+ expected: Number.isFinite(expected) ? expected : null,
1731
+ actual: Number.isFinite(actual) ? actual : null,
1732
+ site,
1733
+ streak,
1734
+ }, '[GRID-PRICE-INVARIANT] Structural resync request failed');
1732
1735
  return streak;
1733
1736
  }
1734
1737
  /**
@@ -1952,9 +1955,9 @@ function buildActionsFromPlan(_bot, plan) {
1952
1955
  // per-action origin there is safe. Other plan origins keep riding
1953
1956
  // non-UPDATE actions unchanged.
1954
1957
  if (action?.type !== COW_ACTIONS.UPDATE) {
1955
- return planOrigin === 'gap-evacuation' ? { ...action } : { ...action, origin: planOrigin };
1958
+ return planOrigin === COW_ORIGINS.GAP_EVACUATION ? { ...action } : { ...action, origin: planOrigin };
1956
1959
  }
1957
- if (planOrigin !== 'gap-evacuation')
1960
+ if (planOrigin !== COW_ORIGINS.GAP_EVACUATION)
1958
1961
  return { ...action, origin: planOrigin };
1959
1962
  if (!hasFrozenGeometry)
1960
1963
  return { ...action, origin: planOrigin };
@@ -3203,8 +3206,8 @@ async function updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seam
3203
3206
  try {
3204
3207
  const actionOrigin = action?.origin;
3205
3208
  const batchOrigin = cowResult?.origin;
3206
- const isCorrectionCreate = actionOrigin === 'spread-correction'
3207
- || (actionOrigin == null && batchOrigin === 'spread-correction');
3209
+ const isCorrectionCreate = actionOrigin === COW_ORIGINS.SPREAD_CORRECTION
3210
+ || (actionOrigin == null && batchOrigin === COW_ORIGINS.SPREAD_CORRECTION);
3208
3211
  if (!isCorrectionCreate) {
3209
3212
  const { check, refreshed } = runLastFillGuardCheck(bot, createPrice, order.size, order.type, lastFillGuardStats, true);
3210
3213
  if (refreshed)
@@ -3375,7 +3378,7 @@ async function updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seam
3375
3378
  try {
3376
3379
  let bypassedEvacuation = false;
3377
3380
  const rotationOrigin = action?.origin;
3378
- if (rotationOrigin === 'gap-evacuation') {
3381
+ if (rotationOrigin === COW_ORIGINS.GAP_EVACUATION) {
3379
3382
  const frozenB = Number(action?.evacBoundary);
3380
3383
  const frozenG = Number(action?.evacGapSlots);
3381
3384
  let stampUsable = Number.isFinite(frozenB) && Number.isFinite(frozenG);
@@ -4054,6 +4057,15 @@ async function updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seam
4054
4057
  });
4055
4058
  }
4056
4059
  const batchResult = await processBatchResults(bot, result, executedContexts);
4060
+ // A `suspended` persist result is a deliberate, temporary gate
4061
+ // (credential-daemon outage), NOT a corrupt/ahead state: the
4062
+ // in-memory master stays authoritative and the dirty-flush
4063
+ // pipeline persists it once suspension lifts. Treat it as a
4064
+ // deferral — never as a structural-resync trigger.
4065
+ const isSuspendedPersist = (r) => {
4066
+ const v = r;
4067
+ return Boolean(v && v.skipped && v.suspended === true);
4068
+ };
4057
4069
  const persistResult = await bot.manager.persistGrid();
4058
4070
  if (persistResult && (persistResult.skipped || persistResult.isValid === false)) {
4059
4071
  bot.manager.logger.log(`[COW][PERSIST-GUARD] First persist attempt was ` +
@@ -4062,7 +4074,21 @@ async function updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seam
4062
4074
  `clearing working grid reference.`, 'warn');
4063
4075
  bot.manager._persistenceWarning = persistResult;
4064
4076
  const retryResult = await bot.manager.persistGrid();
4065
- if (retryResult && (retryResult.skipped || retryResult.isValid === false)) {
4077
+ if (isSuspendedPersist(retryResult)) {
4078
+ bot.manager.logger.log(`[COW][PERSIST-GUARD] Grid persistence is suspended ` +
4079
+ `(${retryResult.reason || 'no reason'}); deferring disk flush until it resumes.`, 'warn');
4080
+ // The committed grid is not on disk yet. Mark it dirty so
4081
+ // the end-of-tick flushGridDirty safety net retries it at
4082
+ // the next end-of-tick flush after the suspension lifts,
4083
+ // instead of relying on the next COW/recovery persist to
4084
+ // converge disk.
4085
+ try {
4086
+ bot.manager._markGridDirty?.();
4087
+ }
4088
+ catch { /* best-effort */ }
4089
+ delete bot.manager._persistenceWarning;
4090
+ }
4091
+ else if (retryResult && (retryResult.skipped || retryResult.isValid === false)) {
4066
4092
  bot.manager.logger.log(`[COW][PERSIST-GUARD] Retry also skipped/invalid ` +
4067
4093
  `(${retryResult.reason || 'no reason'}). Master grid in memory ` +
4068
4094
  `is ahead of disk snapshot; structural resync requested.`, 'error');
@@ -4753,8 +4779,8 @@ function runFinalPivotGate(bot, operations, opContexts, opts = {}) {
4753
4779
  // Spread-correction CREATE bypass (mirrors the build loop:
4754
4780
  // per-action origin, batch origin as back-compat fallback).
4755
4781
  const actionOrigin = action?.origin;
4756
- if (actionOrigin === 'spread-correction'
4757
- || (actionOrigin == null && batchOrigin === 'spread-correction')) {
4782
+ if (actionOrigin === COW_ORIGINS.SPREAD_CORRECTION
4783
+ || (actionOrigin == null && batchOrigin === COW_ORIGINS.SPREAD_CORRECTION)) {
4758
4784
  if (stats)
4759
4785
  stats.bypassed = (Number(stats.bypassed) || 0) + 1;
4760
4786
  continue;
@@ -4806,7 +4832,7 @@ function runFinalPivotGate(bot, operations, opContexts, opts = {}) {
4806
4832
  // The gate must not invent a block it cannot prove —
4807
4833
  // especially not on an op the build loop already allowed.
4808
4834
  const rotOrigin = action?.origin;
4809
- if (rotOrigin === 'gap-evacuation'
4835
+ if (rotOrigin === COW_ORIGINS.GAP_EVACUATION
4810
4836
  && Number.isFinite(Number(action?.evacBoundary))
4811
4837
  && Number.isFinite(Number(action?.evacGapSlots))) {
4812
4838
  if (stats)
@@ -16,13 +16,6 @@ interface CredentialClientOptions {
16
16
  botHmacSecret?: string | null;
17
17
  batchId?: string | null;
18
18
  nodeUrl?: string;
19
- /**
20
- * Accepted for interface compatibility only. The daemon cycles its own
21
- * node list internally, and an uncertain broadcast must never be re-sent
22
- * (a landed tx would be duplicated), so the client no longer cycles
23
- * through fallbacks. Callers may keep passing this field; it is ignored.
24
- */
25
- fallbackNodes?: string[];
26
19
  /**
27
20
  * Fired with the node the daemon reports as in play when a broadcast ends
28
21
  * uncertain (BROADCAST_DEADLINE / outer timeout). The daemon echoes the
@@ -134,8 +134,7 @@ async function executeOperationsViaCredentialDaemon(accountName, operations, opt
134
134
  // (connect/send errors); an uncertain broadcast (RPC timeout, dropped
135
135
  // connection, 25s inner BROADCAST_DEADLINE) is never re-signed, and
136
136
  // re-sending the ops here is unsafe: an uncertain broadcast may have
137
- // landed, duplicating on-chain orders. `fallbackNodes` is kept for
138
- // interface compatibility but never cycled — the recovery layers (COW
137
+ // landed, duplicating on-chain orders. The recovery layers (COW
139
138
  // retry verification, startup adoption, broadcast reconciliation)
140
139
  // verify chain inclusion before any re-broadcast. onNodeFailed fires for
141
140
  // the node the daemon reports as actually in play (the daemon echoes it in
@@ -1,4 +1,3 @@
1
- /** Fill processing runtime - handles order fill events and replay-safe accounting */
2
1
  import * as chainOrders from './chain_orders.js';
3
2
  import type { BotLike, ManagerLogger, IncomingFill } from './types.js';
4
3
  type ChainOrdersLike = typeof chainOrders;
@@ -115,7 +114,7 @@ declare function applyReplaySafeFillAccounting(bot: BotLike, fill: unknown, fill
115
114
  usedFallbackKey?: undefined;
116
115
  }>;
117
116
  /** Shared options for the replay-safe accounting wrappers. */
118
- interface ReplaySafeFillAccountingOptions {
117
+ export interface ReplaySafeFillAccountingOptions {
119
118
  context?: string;
120
119
  logger?: Partial<ManagerLogger>;
121
120
  replayMessage?: ReplayMessageFn;
@@ -1,6 +1,5 @@
1
1
  /** Fill processing runtime - handles order fill events and replay-safe accounting */
2
- import { createRequire } from 'node:module';
3
- const require = createRequire(import.meta.url);
2
+ import { lazyShim } from './lazy_shim.js';
4
3
  import * as chainOrders from './chain_orders.js';
5
4
  import { PROCESSED_FILL_PERSISTENCE_MODES } from './order/processed_fill_store.js';
6
5
  import { NATIVE_CLIENT, FILL_PROCESSING, TIMING, MAINTENANCE, ORDER_TYPES } from './constants.js';
@@ -8,10 +7,10 @@ import { getErrorMessage, resolveSeamMsOrNull, getErrorField } from './utils/err
8
7
  import { isOrderDoesNotExistError } from './dexbot_maintenance_runtime.js';
9
8
  import { slotIndexForPrice, isChainPriceOutOfGrid, isSlotInRail } from './order/utils/math.js';
10
9
  import { ORDER_STATES } from './constants.js';
11
- function buildFillKey(...args) { return require('./order/utils/order').buildFillKey(...args); }
12
- function correctAllPriceMismatches(...args) { return require('./order/utils/order').correctAllPriceMismatches(...args); }
13
- function parseChainOrder(...args) { return require('./order/utils/order').parseChainOrder(...args); }
14
- function retryPersistenceIfNeeded(...args) { return require('./order/utils/system').retryPersistenceIfNeeded(...args); }
10
+ const buildFillKey = lazyShim('./order/utils/order', 'buildFillKey');
11
+ const correctAllPriceMismatches = lazyShim('./order/utils/order', 'correctAllPriceMismatches');
12
+ const parseChainOrder = lazyShim('./order/utils/order', 'parseChainOrder');
13
+ const retryPersistenceIfNeeded = lazyShim('./order/utils/system', 'retryPersistenceIfNeeded');
15
14
  const { readOpenOrdersGuarded } = chainOrders;
16
15
  /**
17
16
  * Whether an unknown fill's order plausibly belongs to THIS bot's grid and
@@ -1,4 +1,4 @@
1
- import type { BotLike, OrderManagerLike, ManagedOrder } from './types.js';
1
+ import type { BotLike, OrderManagerLike, ManagedOrder, SpreadCorrectionStall } from './types.js';
2
2
  /** Loose JSON-object view used across the maintenance runtime. */
3
3
  type JsonObj = Record<string, unknown>;
4
4
  /** A stranded/held chain order as seen by the deferred-hold helpers. */
@@ -11,7 +11,7 @@ interface HeldChainOrder {
11
11
  [key: string]: unknown;
12
12
  }
13
13
  import { usesAmaGridPrice } from './grid_price_source.js';
14
- declare function isWrapperAdapterOwner(...args: unknown[]): any;
14
+ declare const isWrapperAdapterOwner: (...args: unknown[]) => any;
15
15
  /**
16
16
  * Check if a bot configuration uses an AMA grid price source.
17
17
  *
@@ -93,6 +93,22 @@ declare function checkAndApplyBotConfigChanges(bot: BotLike, context?: string, p
93
93
  * @returns {number|null} Live reserve count, or null when unclassifiable
94
94
  */
95
95
  declare function countLiveReserveOrders(manager: OrderManagerLike, config: JsonObj, type: unknown): number | null;
96
+ /**
97
+ * Mirror a spread-check stall into the flags the targeted-sync gate reads.
98
+ *
99
+ * ANY stall — fund, batch, or structural — arms the refresh. When the
100
+ * correction cannot act on the local snapshot, that snapshot may simply be
101
+ * stale relative to the chain: a missed fill leaves an ACTIVE slot occupying
102
+ * the window edge, so `isSlotAvailable` hides every tightening slot and the
103
+ * check reports 'no-candidates' even though real geometry exists. A refresh is
104
+ * the only way to tell a genuinely structural stall from a stale one, so we
105
+ * sync rather than wait for the out-of-spread staleness watchdog to re-center.
106
+ * The stall cause is kept alongside the flag so the log states the truth.
107
+ * Exported for tests.
108
+ * @param {BotLike} bot
109
+ * @param {SpreadCorrectionStall|null|undefined} stall
110
+ */
111
+ export declare function mirrorSpreadStall(bot: BotLike, stall: SpreadCorrectionStall | null | undefined): void;
96
112
  declare function getTargetedSyncReason(bot: BotLike): {
97
113
  reason: string;
98
114
  targetBuy: any;
@@ -509,7 +525,7 @@ declare function syncOpenOrdersAndProcessFills(bot: BotLike, tag: string): Promi
509
525
  export { loadBotsConfigSnapshot, isWrapperAdapterOwner, checkAndApplyBotConfigChanges, buildBotConfigFingerprint, refreshDynamicWeightDistribution, performGridResync, updateBotGridResetMetadata, handlePendingTriggerReset, setupTriggerFileDetection, performPeriodicGridChecks, isOpenOrdersSyncLoopEnabled, startOpenOrdersSyncLoop, stopOpenOrdersSyncLoop, setupBlockchainFetchInterval, stopBlockchainFetchInterval, setupBotsConfigPollInterval, stopBotsConfigPollInterval, executeMaintenanceLogic, getTargetedSyncReason, maybeRunTargetedDriftReconciliation, countLiveReserveOrders, cancelDustOrders, isOrderDoesNotExistError, runGridMaintenance, releaseMarketAdapterRuntime, syncMarketAdapterOnPeriodicConfigCheck, usesAmaGridPrice, runDustHealthCheck, setupDustHealthCheckInterval, requestGridReset, wireStructuralGridResyncRequest, getPipelineSignals, markGridActivity, getMetrics, syncOpenOrdersAndProcessFills, shouldDeferMaintenanceForBroadcast, checkGridLockHoldDuration };
510
526
  declare const _default: {
511
527
  loadBotsConfigSnapshot: typeof loadBotsConfigSnapshot;
512
- isWrapperAdapterOwner: typeof isWrapperAdapterOwner;
528
+ isWrapperAdapterOwner: (...args: unknown[]) => any;
513
529
  checkAndApplyBotConfigChanges: typeof checkAndApplyBotConfigChanges;
514
530
  buildBotConfigFingerprint: typeof buildBotConfigFingerprint;
515
531
  refreshDynamicWeightDistribution: typeof refreshDynamicWeightDistribution;
@@ -527,6 +543,7 @@ declare const _default: {
527
543
  stopBotsConfigPollInterval: typeof stopBotsConfigPollInterval;
528
544
  executeMaintenanceLogic: typeof executeMaintenanceLogic;
529
545
  getTargetedSyncReason: typeof getTargetedSyncReason;
546
+ mirrorSpreadStall: typeof mirrorSpreadStall;
530
547
  countLiveReserveOrders: typeof countLiveReserveOrders;
531
548
  maybeRunTargetedDriftReconciliation: typeof maybeRunTargetedDriftReconciliation;
532
549
  cancelDustOrders: typeof cancelDustOrders;