dexbot 1.4.24 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +7 -2
- package/analysis/README.md +61 -5
- package/analysis/ama_fitting/package.json +1 -1
- package/analysis/analyze_derivatives.ts +1 -1
- package/analysis/bot_key_utils.ts +272 -3
- package/analysis/bot_usage/discover_bot_accounts.ts +64 -9
- package/analysis/bot_usage/kibana_bot_queries.ts +40 -1
- package/analysis/chart_utils.ts +6 -3
- package/analysis/grid_correction_check.ts +932 -0
- package/analysis/resolve_bot_accounts.ts +199 -0
- package/analysis/trade_profitability.ts +36 -8
- package/analysis/tradingview/README.md +25 -0
- package/analysis/tradingview/analyze_tradingview.ts +2 -2
- package/analysis/trend_detection/package.json +1 -1
- package/claw/modules/claw_launcher.ts +5 -4
- package/claw/modules/credit_runtime_adapter.ts +2 -1
- package/claw/modules/dexbot_profiles.ts +4 -3
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/tests/test_claw_mcp_transport.ts +2 -2
- package/dist/analysis/analyze_derivatives.js +1 -1
- package/dist/analysis/bot_key_utils.d.ts +28 -1
- package/dist/analysis/bot_key_utils.d.ts.map +1 -1
- package/dist/analysis/bot_key_utils.js +385 -3
- package/dist/analysis/bot_key_utils.js.map +1 -1
- package/dist/analysis/bot_usage/discover_bot_accounts.js +60 -10
- package/dist/analysis/bot_usage/discover_bot_accounts.js.map +1 -1
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts +45 -1
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts.map +1 -1
- package/dist/analysis/bot_usage/kibana_bot_queries.js +39 -1
- package/dist/analysis/bot_usage/kibana_bot_queries.js.map +1 -1
- package/dist/analysis/chart_utils.d.ts +2 -1
- package/dist/analysis/chart_utils.d.ts.map +1 -1
- package/dist/analysis/chart_utils.js +4 -1
- package/dist/analysis/chart_utils.js.map +1 -1
- package/dist/analysis/grid_correction_check.d.ts +90 -0
- package/dist/analysis/grid_correction_check.d.ts.map +1 -0
- package/dist/analysis/grid_correction_check.js +901 -0
- package/dist/analysis/grid_correction_check.js.map +1 -0
- package/dist/analysis/resolve_bot_accounts.d.ts +3 -0
- package/dist/analysis/resolve_bot_accounts.d.ts.map +1 -0
- package/dist/analysis/resolve_bot_accounts.js +216 -0
- package/dist/analysis/resolve_bot_accounts.js.map +1 -0
- package/dist/analysis/trade_profitability.d.ts.map +1 -1
- package/dist/analysis/trade_profitability.js +40 -8
- package/dist/analysis/trade_profitability.js.map +1 -1
- package/dist/analysis/tradingview/analyze_tradingview.js +2 -2
- package/dist/analysis/tradingview/analyze_tradingview.js.map +1 -1
- package/dist/bot.d.ts.map +1 -1
- package/dist/bot.js +2 -1
- package/dist/bot.js.map +1 -1
- package/dist/dexbot.d.ts.map +1 -1
- package/dist/dexbot.js +134 -22
- package/dist/dexbot.js.map +1 -1
- package/dist/market_adapter/ama_signal_runner.js +2 -2
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +2 -1
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js.map +1 -1
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +4 -1
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts.map +1 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js +3 -2
- package/dist/market_adapter/inputs/fetch_lp_data.js.map +1 -1
- package/dist/modules/account_bots.d.ts +42 -1
- package/dist/modules/account_bots.d.ts.map +1 -1
- package/dist/modules/account_bots.js +221 -22
- package/dist/modules/account_bots.js.map +1 -1
- package/dist/modules/account_orders.d.ts +7 -10
- package/dist/modules/account_orders.d.ts.map +1 -1
- package/dist/modules/account_orders.js +26 -2
- package/dist/modules/account_orders.js.map +1 -1
- package/dist/modules/bitshares_client.d.ts +4 -1
- package/dist/modules/bitshares_client.d.ts.map +1 -1
- package/dist/modules/bitshares_client.js +12 -4
- package/dist/modules/bitshares_client.js.map +1 -1
- package/dist/modules/bot_settings.d.ts.map +1 -1
- package/dist/modules/bot_settings.js +2 -1
- package/dist/modules/bot_settings.js.map +1 -1
- package/dist/modules/chain_orders.d.ts.map +1 -1
- package/dist/modules/chain_orders.js +21 -2
- package/dist/modules/chain_orders.js.map +1 -1
- package/dist/modules/cli_colors.d.ts +39 -0
- package/dist/modules/cli_colors.d.ts.map +1 -0
- package/dist/modules/cli_colors.js +42 -0
- package/dist/modules/cli_colors.js.map +1 -0
- package/dist/modules/config.d.ts.map +1 -1
- package/dist/modules/config.js +27 -0
- package/dist/modules/config.js.map +1 -1
- package/dist/modules/constants.d.ts +14 -1
- package/dist/modules/constants.d.ts.map +1 -1
- package/dist/modules/constants.js +66 -11
- package/dist/modules/constants.js.map +1 -1
- package/dist/modules/credit_pricing.d.ts +61 -0
- package/dist/modules/credit_pricing.d.ts.map +1 -0
- package/dist/modules/credit_pricing.js +237 -0
- package/dist/modules/credit_pricing.js.map +1 -0
- package/dist/modules/credit_runtime.d.ts +1 -0
- package/dist/modules/credit_runtime.d.ts.map +1 -1
- package/dist/modules/credit_runtime.js +49 -107
- package/dist/modules/credit_runtime.js.map +1 -1
- package/dist/modules/dexbot_class.d.ts +25 -0
- package/dist/modules/dexbot_class.d.ts.map +1 -1
- package/dist/modules/dexbot_class.js +82 -12
- package/dist/modules/dexbot_class.js.map +1 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +39 -17
- package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_cow_runtime.js +875 -134
- package/dist/modules/dexbot_cow_runtime.js.map +1 -1
- package/dist/modules/dexbot_fill_runtime.d.ts +11 -1
- package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_fill_runtime.js +108 -1
- package/dist/modules/dexbot_fill_runtime.js.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts +29 -14
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +279 -63
- package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
- package/dist/modules/dexbot_startup_runtime.d.ts +1 -0
- package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_startup_runtime.js +58 -7
- package/dist/modules/dexbot_startup_runtime.js.map +1 -1
- package/dist/modules/dexbot_state_recovery.d.ts.map +1 -1
- package/dist/modules/dexbot_state_recovery.js +55 -3
- package/dist/modules/dexbot_state_recovery.js.map +1 -1
- package/dist/modules/fund_registry.d.ts +1 -1
- package/dist/modules/fund_registry.js +1 -1
- package/dist/modules/graceful_shutdown.d.ts +12 -1
- package/dist/modules/graceful_shutdown.d.ts.map +1 -1
- package/dist/modules/graceful_shutdown.js +39 -1
- package/dist/modules/graceful_shutdown.js.map +1 -1
- package/dist/modules/key_store.d.ts.map +1 -1
- package/dist/modules/key_store.js +38 -1
- package/dist/modules/key_store.js.map +1 -1
- package/dist/modules/launcher/adapter_requirement.d.ts +38 -0
- package/dist/modules/launcher/adapter_requirement.d.ts.map +1 -0
- package/dist/modules/launcher/adapter_requirement.js +141 -0
- package/dist/modules/launcher/adapter_requirement.js.map +1 -0
- package/dist/modules/launcher/bot_supervisor.d.ts +2 -2
- package/dist/modules/launcher/bot_supervisor.d.ts.map +1 -1
- package/dist/modules/launcher/bot_supervisor.js +32 -3
- package/dist/modules/launcher/bot_supervisor.js.map +1 -1
- package/dist/modules/launcher/launch_modes.d.ts.map +1 -1
- package/dist/modules/launcher/launch_modes.js +5 -4
- package/dist/modules/launcher/launch_modes.js.map +1 -1
- package/dist/modules/launcher/monolithic_runtime.d.ts +1 -1
- package/dist/modules/launcher/monolithic_runtime.d.ts.map +1 -1
- package/dist/modules/launcher/monolithic_runtime.js +12 -8
- package/dist/modules/launcher/monolithic_runtime.js.map +1 -1
- package/dist/modules/launcher/status_reporting.d.ts +6 -6
- package/dist/modules/launcher/status_reporting.d.ts.map +1 -1
- package/dist/modules/launcher/status_reporting.js +9 -6
- package/dist/modules/launcher/status_reporting.js.map +1 -1
- package/dist/modules/order/accounting.d.ts.map +1 -1
- package/dist/modules/order/accounting.js +20 -3
- package/dist/modules/order/accounting.js.map +1 -1
- package/dist/modules/order/export.d.ts +1 -1
- package/dist/modules/order/export.js +1 -1
- package/dist/modules/order/grid.d.ts +1 -1
- package/dist/modules/order/grid.d.ts.map +1 -1
- package/dist/modules/order/grid.js +335 -43
- package/dist/modules/order/grid.js.map +1 -1
- package/dist/modules/order/grid_reconcile.d.ts +4 -2
- package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile.js +242 -222
- package/dist/modules/order/grid_reconcile.js.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts +11 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.js +132 -6
- package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
- package/dist/modules/order/logger.d.ts +23 -1
- package/dist/modules/order/logger.d.ts.map +1 -1
- package/dist/modules/order/logger.js +60 -17
- package/dist/modules/order/logger.js.map +1 -1
- package/dist/modules/order/manager.d.ts +68 -14
- package/dist/modules/order/manager.d.ts.map +1 -1
- package/dist/modules/order/manager.js +281 -52
- package/dist/modules/order/manager.js.map +1 -1
- package/dist/modules/order/strategy.d.ts.map +1 -1
- package/dist/modules/order/strategy.js +25 -8
- package/dist/modules/order/strategy.js.map +1 -1
- package/dist/modules/order/sync_engine.d.ts.map +1 -1
- package/dist/modules/order/sync_engine.js +262 -167
- package/dist/modules/order/sync_engine.js.map +1 -1
- package/dist/modules/order/utils/math.d.ts +47 -5
- package/dist/modules/order/utils/math.d.ts.map +1 -1
- package/dist/modules/order/utils/math.js +175 -11
- package/dist/modules/order/utils/math.js.map +1 -1
- package/dist/modules/order/utils/order.d.ts +16 -15
- package/dist/modules/order/utils/order.d.ts.map +1 -1
- package/dist/modules/order/utils/order.js +199 -32
- package/dist/modules/order/utils/order.js.map +1 -1
- package/dist/modules/order/utils/slot.d.ts +8 -0
- package/dist/modules/order/utils/slot.d.ts.map +1 -0
- package/dist/modules/order/utils/slot.js +16 -0
- package/dist/modules/order/utils/slot.js.map +1 -0
- package/dist/modules/order/utils/system.d.ts +52 -4
- package/dist/modules/order/utils/system.d.ts.map +1 -1
- package/dist/modules/order/utils/system.js +109 -16
- package/dist/modules/order/utils/system.js.map +1 -1
- package/dist/modules/order/utils/validate.d.ts +6 -9
- package/dist/modules/order/utils/validate.d.ts.map +1 -1
- package/dist/modules/order/utils/validate.js +107 -85
- package/dist/modules/order/utils/validate.js.map +1 -1
- package/dist/modules/order/working_grid.d.ts +5 -0
- package/dist/modules/order/working_grid.d.ts.map +1 -1
- package/dist/modules/order/working_grid.js +21 -0
- package/dist/modules/order/working_grid.js.map +1 -1
- package/dist/modules/paths.d.ts +4 -2
- package/dist/modules/paths.d.ts.map +1 -1
- package/dist/modules/paths.js +79 -24
- package/dist/modules/paths.js.map +1 -1
- package/dist/modules/utils/chain_logs.d.ts +11 -0
- package/dist/modules/utils/chain_logs.d.ts.map +1 -0
- package/dist/modules/utils/chain_logs.js +23 -0
- package/dist/modules/utils/chain_logs.js.map +1 -0
- package/dist/modules/utils/sanitize_key.d.ts +7 -0
- package/dist/modules/utils/sanitize_key.d.ts.map +1 -1
- package/dist/modules/utils/sanitize_key.js +15 -0
- package/dist/modules/utils/sanitize_key.js.map +1 -1
- package/dist/pm2.d.ts +11 -1
- package/dist/pm2.d.ts.map +1 -1
- package/dist/pm2.js +66 -15
- package/dist/pm2.js.map +1 -1
- package/dist/scripts/analyze-credit.d.ts +4 -0
- package/dist/scripts/analyze-credit.d.ts.map +1 -0
- package/dist/scripts/analyze-credit.js +760 -0
- package/dist/scripts/analyze-credit.js.map +1 -0
- package/dist/scripts/analyze-orders.d.ts +1 -10
- package/dist/scripts/analyze-orders.d.ts.map +1 -1
- package/dist/scripts/analyze-orders.js +34 -36
- package/dist/scripts/analyze-orders.js.map +1 -1
- package/dist/scripts/generate_market_adapter_whitelist.d.ts +2 -5
- package/dist/scripts/generate_market_adapter_whitelist.d.ts.map +1 -1
- package/dist/scripts/generate_market_adapter_whitelist.js +83 -1
- package/dist/scripts/generate_market_adapter_whitelist.js.map +1 -1
- package/dist/scripts/test-credit-renewal.d.ts.map +1 -1
- package/dist/scripts/test-credit-renewal.js +37 -5
- package/dist/scripts/test-credit-renewal.js.map +1 -1
- package/dist/scripts/tv.d.ts +11 -0
- package/dist/scripts/tv.d.ts.map +1 -0
- package/dist/scripts/tv.js +357 -0
- package/dist/scripts/tv.js.map +1 -0
- package/dist/scripts/update.js +77 -10
- package/dist/scripts/update.js.map +1 -1
- package/dist/unlock.d.ts.map +1 -1
- package/dist/unlock.js +20 -8
- package/dist/unlock.js.map +1 -1
- package/docs/DEXBOT_COMPARISON.md +11 -11
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_RECALCULATION.md +6 -0
- package/docs/MPA_CREDIT_USAGE.md +3 -1
- package/docs/README.md +2 -2
- package/docs/WORKFLOW.md +3 -1
- package/market_adapter/README.md +10 -0
- package/package.json +3 -1
- package/scripts/README.md +16 -0
|
@@ -7,7 +7,6 @@ const __dirname = _esmDirname(__filename);
|
|
|
7
7
|
const require = createRequire(import.meta.url);
|
|
8
8
|
import fs from 'node:fs';
|
|
9
9
|
import { spawn } from 'node:child_process';
|
|
10
|
-
import { createHash } from './crypto/sync.js';
|
|
11
10
|
import { nowIso } from './order/utils/system.js';
|
|
12
11
|
import { path } from './path_api.js';
|
|
13
12
|
import * as chainOrders from './chain_orders.js';
|
|
@@ -27,6 +26,7 @@ import { BroadcastUncertainError } from './dexbot_credential_client.js';
|
|
|
27
26
|
import * as configModule from './config.js';
|
|
28
27
|
const { Config } = configModule;
|
|
29
28
|
import { getErrorMessage } from './utils/errors.js';
|
|
29
|
+
import { isSameBotName } from './utils/sanitize_key.js';
|
|
30
30
|
import { usesAmaGridPrice } from './grid_price_source.js';
|
|
31
31
|
function hasOpenOrdersSyncLoopMsSet(...args) { return require('./config').hasOpenOrdersSyncLoopMsSet(...args); }
|
|
32
32
|
function getOpenOrdersSyncLoopMs(...args) { return require('./config').getOpenOrdersSyncLoopMs(...args); }
|
|
@@ -41,6 +41,8 @@ function loadAmaCenterSnapshot(...args) { return require('./order/utils/system')
|
|
|
41
41
|
function sleep(...args) { return require('./order/utils/system').sleep(...args); }
|
|
42
42
|
function parseJsonWithComments(...args) { return require('./order/utils/system').parseJsonWithComments(...args); }
|
|
43
43
|
function isPm2Runtime(...args) { return require('./order/logger').isPm2Runtime(...args); }
|
|
44
|
+
function isWrapperAdapterOwner(...args) { return require('./launcher/adapter_requirement').isWrapperAdapterOwner(...args); }
|
|
45
|
+
function readAdapterRequirement(...args) { return require('./launcher/adapter_requirement').readAdapterRequirement(...args); }
|
|
44
46
|
function getSharedMarketAdapterRuntime(...args) { return require('./launcher/market_adapter_runtime').getSharedMarketAdapterRuntime(...args); }
|
|
45
47
|
function resetMarketAdapterWhitelistCache(...args) { return require('./market_adapter_whitelist').resetMarketAdapterWhitelistCache(...args); }
|
|
46
48
|
function isBotDynamicWeightWhitelisted(...args) { return require('./market_adapter_whitelist').isBotDynamicWeightWhitelisted(...args); }
|
|
@@ -51,6 +53,9 @@ function calculateSwapInAmount(...args) { return require('./order/utils/math').c
|
|
|
51
53
|
function floatToBlockchainInt(...args) { return require('./order/utils/math').floatToBlockchainInt(...args); }
|
|
52
54
|
function blockchainToFloat(...args) { return require('./order/utils/math').blockchainToFloat(...args); }
|
|
53
55
|
function updateDynamicGridSnapshotSync(...args) { return require('../market_adapter/utils/dynamic_grid_snapshot').updateDynamicGridSnapshotSync(...args); }
|
|
56
|
+
// Lazy require: dexbot_fill_runtime imports this module, so a static import
|
|
57
|
+
// would be circular; at call time both modules are fully loaded.
|
|
58
|
+
function scheduleFillConsumerRestartFn(...args) { return require('./dexbot_fill_runtime').scheduleFillConsumerRestart(...args); }
|
|
54
59
|
function reconcileGridOrders(...args) { return require('./order/grid_reconcile').reconcileGridOrders(...args); }
|
|
55
60
|
function formatUnmatchedChainOrder(...args) { return require('./order/utils/order').formatUnmatchedChainOrder(...args); }
|
|
56
61
|
function getSideBudget(...args) { return require('./order/utils/order').getSideBudget(...args); }
|
|
@@ -94,7 +99,7 @@ const GRID_RESYNC_REASONS = Object.freeze({
|
|
|
94
99
|
...MARKET_ADAPTER_TRIGGER_RESETS,
|
|
95
100
|
manual_grid_resync: MANUAL_TRIGGER_METADATA,
|
|
96
101
|
rms_structural_grid_resync: {
|
|
97
|
-
shouldRefreshCenterPrice:
|
|
102
|
+
shouldRefreshCenterPrice: false,
|
|
98
103
|
centerRefreshContext: 'RMS structural grid resync',
|
|
99
104
|
centerRefreshLabel: 'RMS structural grid resync',
|
|
100
105
|
},
|
|
@@ -252,37 +257,13 @@ async function maybeRunTargetedDriftReconciliation(bot, context) {
|
|
|
252
257
|
}
|
|
253
258
|
/**
|
|
254
259
|
* Load and fingerprint the bots.json configuration file.
|
|
260
|
+
* Delegates to the canonical launcher/adapter_requirement helper so the
|
|
261
|
+
* fingerprint is semantic (AMA-relevant changes only) and identical to the
|
|
262
|
+
* unlock wrapper watchdog's view of the same file.
|
|
255
263
|
* @returns {any} Snapshot with exists flag, fingerprint, active bots list, and adapter requirement
|
|
256
264
|
*/
|
|
257
265
|
function loadBotsConfigSnapshot() {
|
|
258
|
-
|
|
259
|
-
return {
|
|
260
|
-
exists: false,
|
|
261
|
-
fingerprint: null,
|
|
262
|
-
activeBots: [],
|
|
263
|
-
needsMarketAdapter: false,
|
|
264
|
-
};
|
|
265
|
-
}
|
|
266
|
-
const raw = storage.readFile(PROFILES_BOTS_FILE);
|
|
267
|
-
if (!raw || !raw.trim()) {
|
|
268
|
-
return {
|
|
269
|
-
exists: false,
|
|
270
|
-
fingerprint: null,
|
|
271
|
-
activeBots: [],
|
|
272
|
-
needsMarketAdapter: false,
|
|
273
|
-
};
|
|
274
|
-
}
|
|
275
|
-
const fingerprint = createHash('sha1').update(raw).digest('hex');
|
|
276
|
-
const parsed = parseJsonWithComments(raw);
|
|
277
|
-
const bots = Array.isArray(parsed?.bots) ? parsed.bots.filter(Boolean) : [];
|
|
278
|
-
const activeBots = bots.filter((bot) => bot.active !== false);
|
|
279
|
-
return {
|
|
280
|
-
exists: true,
|
|
281
|
-
fingerprint,
|
|
282
|
-
config: parsed,
|
|
283
|
-
activeBots,
|
|
284
|
-
needsMarketAdapter: activeBots.some(usesAmaGridPrice),
|
|
285
|
-
};
|
|
266
|
+
return readAdapterRequirement(PROFILES_BOTS_FILE);
|
|
286
267
|
}
|
|
287
268
|
/**
|
|
288
269
|
* Parse PM2 jlist command output to extract process names.
|
|
@@ -376,6 +357,16 @@ async function syncMarketAdapterOnPeriodicConfigCheck(bot, context = 'periodic')
|
|
|
376
357
|
if (typeof bot._syncMarketAdapterHook === 'function') {
|
|
377
358
|
return await bot._syncMarketAdapterHook(context);
|
|
378
359
|
}
|
|
360
|
+
// Centralized ownership: in monolithic mode the unlock wrapper watchdog
|
|
361
|
+
// (shared 1min tick) is the sole market-adapter spawner. A per-bot sync here would
|
|
362
|
+
// re-read bots.json and re-drive the adapter N times (once per bot) and
|
|
363
|
+
// fight the wrapper over the adapter child, so bots skip it entirely and
|
|
364
|
+
// act as pure adapter-output consumers. Wrapper-less modes (dexbot test
|
|
365
|
+
// one-shot, isolated supervisor, PM2) keep the in-bot fallback below.
|
|
366
|
+
// Checked before any file I/O so the skip also removes the duplicate read.
|
|
367
|
+
if (!isPm2Runtime() && isWrapperAdapterOwner()) {
|
|
368
|
+
return { skipped: true, reason: 'wrapper-owned' };
|
|
369
|
+
}
|
|
379
370
|
if (bot._marketAdapterWatchdogInFlight) {
|
|
380
371
|
return { skipped: true, reason: 'in-flight' };
|
|
381
372
|
}
|
|
@@ -384,7 +375,19 @@ async function syncMarketAdapterOnPeriodicConfigCheck(bot, context = 'periodic')
|
|
|
384
375
|
const snapshot = typeof bot._loadBotsConfigSnapshot === 'function'
|
|
385
376
|
? await bot._loadBotsConfigSnapshot()
|
|
386
377
|
: loadBotsConfigSnapshot();
|
|
387
|
-
|
|
378
|
+
// Corrupt/unreadable bots.json must not move the adapter: previously
|
|
379
|
+
// the parse/read throw aborted the check and left the adapter alone.
|
|
380
|
+
// Keep that behavior by skipping the sync without touching the stored
|
|
381
|
+
// fingerprint, so the next tick re-evaluates once the file is valid.
|
|
382
|
+
if (snapshot.corrupt || snapshot.readError) {
|
|
383
|
+
const reason = snapshot.corrupt ? 'corrupt-config' : 'unreadable-config';
|
|
384
|
+
const detail = snapshot.corrupt ? 'corrupt' : 'unreadable';
|
|
385
|
+
bot._warn(`Ignoring ${detail} bots.json during ${context}; keeping previous market adapter state.`);
|
|
386
|
+
return { skipped: true, reason };
|
|
387
|
+
}
|
|
388
|
+
// ?? (not ||): the semantic fingerprint is legitimately '' when no
|
|
389
|
+
// active AMA bot exists, and '' must compare equal to a stored ''.
|
|
390
|
+
const previousFingerprint = bot._marketAdapterWatchdogFingerprint ?? null;
|
|
388
391
|
const changed = snapshot.fingerprint !== previousFingerprint;
|
|
389
392
|
bot._marketAdapterWatchdogFingerprint = snapshot.fingerprint;
|
|
390
393
|
if (changed) {
|
|
@@ -739,7 +742,7 @@ function performGridResync(bot, options = {}) {
|
|
|
739
742
|
const content = storage.readFile(PROFILES_BOTS_FILE);
|
|
740
743
|
const allBotsConfig = parseJsonWithComments(content).bots || [];
|
|
741
744
|
const myName = self.config.name;
|
|
742
|
-
const updatedBot = allBotsConfig.find((b) => b.name
|
|
745
|
+
const updatedBot = allBotsConfig.find((b) => isSameBotName(b.name, myName));
|
|
743
746
|
if (updatedBot) {
|
|
744
747
|
self._log(`Reloaded configuration for bot '${myName}'`);
|
|
745
748
|
const oldKey = self.config.botKey;
|
|
@@ -782,14 +785,38 @@ function performGridResync(bot, options = {}) {
|
|
|
782
785
|
// of virtualizing live ACTIVE slots; the trigger file is retained
|
|
783
786
|
// so the resync retries on a clean read.
|
|
784
787
|
let resyncReadAmbiguous = false;
|
|
788
|
+
const readOpenOrdersForResync = (label) => readOpenOrdersGuarded(chainOrders, self.accountId, {
|
|
789
|
+
log: (message, level) => self._log(message, level),
|
|
790
|
+
label,
|
|
791
|
+
detail: 'trigger-file resync',
|
|
792
|
+
});
|
|
793
|
+
// Empty-read confirm guard: an EMPTY read during a trigger reset is
|
|
794
|
+
// only accepted after one confirming re-read. A single 0-order
|
|
795
|
+
// snapshot from a lagging/partial node must never wipe a live grid
|
|
796
|
+
// and rebuild over existing orders (phantom reset — duplicated
|
|
797
|
+
// price levels and mis-tracked orders). A contradicted re-read
|
|
798
|
+
// (non-empty) feeds the fresh snapshot to the resync so the rebuild
|
|
799
|
+
// reconciles against the real chain state instead of overwriting it.
|
|
785
800
|
const readFn = async () => {
|
|
786
|
-
const orders = await
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
}
|
|
791
|
-
if (orders
|
|
801
|
+
const orders = await readOpenOrdersForResync('GRID-RESYNC');
|
|
802
|
+
if (orders === null) {
|
|
803
|
+
resyncReadAmbiguous = true;
|
|
804
|
+
return orders;
|
|
805
|
+
}
|
|
806
|
+
if (orders.length > 0)
|
|
807
|
+
return orders;
|
|
808
|
+
await sleep(TIMING.SYNC_EMPTY_READ_CONFIRM_DELAY_MS);
|
|
809
|
+
const confirmed = await readOpenOrdersForResync('GRID-RESYNC-CONFIRM');
|
|
810
|
+
if (confirmed === null) {
|
|
792
811
|
resyncReadAmbiguous = true;
|
|
812
|
+
return orders;
|
|
813
|
+
}
|
|
814
|
+
if (confirmed.length > 0) {
|
|
815
|
+
self._log(`[GRID-RESYNC] Empty open-order read contradicted by confirm re-read ` +
|
|
816
|
+
`(${confirmed.length} order(s) present) — using fresh non-empty snapshot`, 'warn');
|
|
817
|
+
return confirmed;
|
|
818
|
+
}
|
|
819
|
+
self._log('[GRID-RESYNC] Empty open-order read confirmed by re-read — accepting empty account', 'info');
|
|
793
820
|
return orders;
|
|
794
821
|
};
|
|
795
822
|
await recalculateGrid(self.manager, {
|
|
@@ -861,8 +888,11 @@ async function handlePendingTriggerReset(bot) {
|
|
|
861
888
|
bot._log('Pending trigger file detected. Processing reset before startup...');
|
|
862
889
|
const triggerInfo = readTriggerMetadata(bot.triggerFile);
|
|
863
890
|
let resetSucceeded = false;
|
|
891
|
+
// skipIdle: a pending trigger is an explicit reset command that must run
|
|
892
|
+
// before the startup sequence. The idle gate can block it indefinitely
|
|
893
|
+
// when the fill queue never drains, which would silently drop the reset.
|
|
864
894
|
await bot.manager._fillProcessingLock.acquire(async () => {
|
|
865
|
-
resetSucceeded = await performGridResync(bot, buildGridResyncOptions(triggerInfo));
|
|
895
|
+
resetSucceeded = await performGridResync(bot, { ...buildGridResyncOptions(triggerInfo), skipIdle: true });
|
|
866
896
|
});
|
|
867
897
|
if (!resetSucceeded) {
|
|
868
898
|
bot._warn('Pending trigger reset failed. Continuing with normal startup path.');
|
|
@@ -907,7 +937,13 @@ async function setupTriggerFileDetection(bot) {
|
|
|
907
937
|
bot.manager._fillProcessingLock.acquire(async () => {
|
|
908
938
|
if (bot._shuttingDown)
|
|
909
939
|
return;
|
|
910
|
-
|
|
940
|
+
// skipIdle: a trigger file is an explicit reset command.
|
|
941
|
+
// Deferring it behind the idle cooldown can loop forever
|
|
942
|
+
// when grid activity is continuous (fills arriving during
|
|
943
|
+
// placement batches refresh the settle window); the fill
|
|
944
|
+
// lock below already serializes the reset with in-flight
|
|
945
|
+
// batches.
|
|
946
|
+
const ok = await performGridResync(bot, { ...buildGridResyncOptions(triggerInfo), skipIdle: true });
|
|
911
947
|
if (!ok) {
|
|
912
948
|
bot._warn('Runtime trigger reset failed; retaining existing grid state.');
|
|
913
949
|
}
|
|
@@ -1003,7 +1039,7 @@ function startOpenOrdersSyncLoop(bot) {
|
|
|
1003
1039
|
if (syncResult?.filledOrders && syncResult.filledOrders.length > 0) {
|
|
1004
1040
|
bot._log(`Open-orders sync loop: ${syncResult.filledOrders.length} grid order(s) found filled on-chain. Triggering rebalance.`, 'info');
|
|
1005
1041
|
bot._markGridActivity?.('open-orders sync fill');
|
|
1006
|
-
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), 'open-orders sync fill rebalance');
|
|
1042
|
+
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), 'open-orders sync fill rebalance', { isReplay: true });
|
|
1007
1043
|
if (!batchResult?.aborted) {
|
|
1008
1044
|
await bot.manager.persistGrid();
|
|
1009
1045
|
}
|
|
@@ -1134,7 +1170,7 @@ function setupBlockchainFetchInterval(bot) {
|
|
|
1134
1170
|
if (syncResult.filledOrders && syncResult.filledOrders.length > 0) {
|
|
1135
1171
|
bot._log(`Periodic sync: ${syncResult.filledOrders.length} grid order(s) found filled on-chain. Triggering rebalance.`, 'info');
|
|
1136
1172
|
bot._markGridActivity?.('periodic sync fill rebalance');
|
|
1137
|
-
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), 'periodic sync fill rebalance');
|
|
1173
|
+
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), 'periodic sync fill rebalance', { isReplay: true });
|
|
1138
1174
|
if (!batchResult?.aborted) {
|
|
1139
1175
|
await bot.manager.persistGrid();
|
|
1140
1176
|
}
|
|
@@ -1180,6 +1216,83 @@ function stopBlockchainFetchInterval(bot) {
|
|
|
1180
1216
|
bot._log('Stopped periodic blockchain fetch interval');
|
|
1181
1217
|
}
|
|
1182
1218
|
}
|
|
1219
|
+
/**
|
|
1220
|
+
* Set up the periodic bots.json fingerprint poll interval.
|
|
1221
|
+
* Fallback config-change detector for wrapper-less modes (dexbot test
|
|
1222
|
+
* one-shot, isolated supervisor, PM2): decoupled from the heavy blockchain
|
|
1223
|
+
* fetch interval (default 240min) so config changes are visible within
|
|
1224
|
+
* BOTS_CONFIG_POLL_INTERVAL_MS (default 1min, shared with the wrapper
|
|
1225
|
+
* watchdog interval).
|
|
1226
|
+
* In monolithic mode the unlock wrapper watchdog already polls bots.json
|
|
1227
|
+
* on the same shared interval as the sole adapter owner, so no per-bot poll
|
|
1228
|
+
* is created here.
|
|
1229
|
+
* Uses the shared fingerprint via syncMarketAdapterOnPeriodicConfigCheck so
|
|
1230
|
+
* the market adapter start/stop logic stays in one place.
|
|
1231
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
1232
|
+
*/
|
|
1233
|
+
function setupBotsConfigPollInterval(bot) {
|
|
1234
|
+
// Allow per-bot timing override (e.g. tests or per-bot tuning)
|
|
1235
|
+
let intervalMs = bot.config?.timing?.BOTS_CONFIG_POLL_INTERVAL_MS;
|
|
1236
|
+
if (!Number.isFinite(intervalMs) || intervalMs <= 0) {
|
|
1237
|
+
intervalMs = Number(TIMING.BOTS_CONFIG_POLL_INTERVAL_MS);
|
|
1238
|
+
}
|
|
1239
|
+
if (!Number.isFinite(intervalMs) || intervalMs <= 0) {
|
|
1240
|
+
bot._log(`Bots-config poll interval disabled (value: ${intervalMs}). Fingerprint changes will only be seen on blockchain fetch ticks.`);
|
|
1241
|
+
return;
|
|
1242
|
+
}
|
|
1243
|
+
if (!isPm2Runtime() && isWrapperAdapterOwner()) {
|
|
1244
|
+
bot._log('Bots-config poll disabled (market adapter owned by unlock wrapper).');
|
|
1245
|
+
return;
|
|
1246
|
+
}
|
|
1247
|
+
// Ensure the fingerprint exists before the first interval fires.
|
|
1248
|
+
// setupBlockchainFetchInterval already fires syncMarketAdapterOnPeriodicConfigCheck
|
|
1249
|
+
// at startup, so this is a fallback for bots that never run the blockchain interval
|
|
1250
|
+
// (e.g. dryRun or accountId missing). Gate on === null (never checked):
|
|
1251
|
+
// '' is a valid checked steady-state and must not re-trigger the pre-seed.
|
|
1252
|
+
if (bot._marketAdapterWatchdogFingerprint === null || bot._marketAdapterWatchdogFingerprint === undefined) {
|
|
1253
|
+
syncMarketAdapterOnPeriodicConfigCheck(bot, 'startup bots-config poll setup').catch((err) => {
|
|
1254
|
+
bot._warn(`Bots-config poll setup failed: ${getErrorMessage(err)}`);
|
|
1255
|
+
});
|
|
1256
|
+
}
|
|
1257
|
+
if (bot._botsConfigPollInterval !== null && bot._botsConfigPollInterval !== undefined) {
|
|
1258
|
+
stopBotsConfigPollInterval(bot);
|
|
1259
|
+
}
|
|
1260
|
+
bot._botsConfigPollInterval = setInterval(async () => {
|
|
1261
|
+
if (bot._shuttingDown)
|
|
1262
|
+
return;
|
|
1263
|
+
// Coalesce with the blockchain-fetch tick: if its tick is already
|
|
1264
|
+
// driving the watchdog, skip to avoid double PM2 queries.
|
|
1265
|
+
if (bot._marketAdapterWatchdogInFlight)
|
|
1266
|
+
return;
|
|
1267
|
+
if (bot._botsConfigPollInFlight)
|
|
1268
|
+
return;
|
|
1269
|
+
bot._botsConfigPollInFlight = true;
|
|
1270
|
+
try {
|
|
1271
|
+
await syncMarketAdapterOnPeriodicConfigCheck(bot, 'bots-config poll');
|
|
1272
|
+
}
|
|
1273
|
+
catch (err) {
|
|
1274
|
+
bot._warn(`Bots-config poll failed: ${getErrorMessage(err)}`);
|
|
1275
|
+
}
|
|
1276
|
+
finally {
|
|
1277
|
+
bot._botsConfigPollInFlight = false;
|
|
1278
|
+
}
|
|
1279
|
+
}, intervalMs);
|
|
1280
|
+
if (typeof bot._botsConfigPollInterval.unref === 'function') {
|
|
1281
|
+
bot._botsConfigPollInterval.unref();
|
|
1282
|
+
}
|
|
1283
|
+
bot._log(`Started bots-config poll interval: every ${Math.round(intervalMs / 1000)}s (fingerprint check)`);
|
|
1284
|
+
}
|
|
1285
|
+
/**
|
|
1286
|
+
* Stop the periodic bots.json fingerprint poll interval.
|
|
1287
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
1288
|
+
*/
|
|
1289
|
+
function stopBotsConfigPollInterval(bot) {
|
|
1290
|
+
if (bot._botsConfigPollInterval !== null && bot._botsConfigPollInterval !== undefined) {
|
|
1291
|
+
clearInterval(bot._botsConfigPollInterval);
|
|
1292
|
+
bot._botsConfigPollInterval = null;
|
|
1293
|
+
bot._log('Stopped bots-config poll interval');
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1183
1296
|
/**
|
|
1184
1297
|
* Release the market adapter runtime for a bot.
|
|
1185
1298
|
* In PM2 mode this is a no-op; in direct mode it calls the shared runtime's releaseBot.
|
|
@@ -1417,7 +1530,13 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
1417
1530
|
if (pendingCorrections > 0) {
|
|
1418
1531
|
const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders);
|
|
1419
1532
|
if (correctionResult.failed > 0) {
|
|
1420
|
-
|
|
1533
|
+
const failedDetails = (correctionResult.results || [])
|
|
1534
|
+
.filter((r) => !(r.result && r.result.success))
|
|
1535
|
+
.slice(0, 3)
|
|
1536
|
+
.map((r) => `${r.chainOrderId ?? '?'}: ${r.result?.error ?? 'unknown'}`)
|
|
1537
|
+
.join(' | ');
|
|
1538
|
+
bot._warn(`[MAINT] ${correctionResult.failed}/${pendingCorrections} price correction(s) failed` +
|
|
1539
|
+
(failedDetails ? ` — ${failedDetails}` : ''));
|
|
1421
1540
|
}
|
|
1422
1541
|
}
|
|
1423
1542
|
// Dust detection runs before the pipeline gate. Cancellation is immediate
|
|
@@ -1532,12 +1651,27 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
1532
1651
|
// committed boundary, and re-plans on fund change, so this tick is
|
|
1533
1652
|
// race-safe whenever it does run.
|
|
1534
1653
|
if (!boundaryShiftPending) {
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1654
|
+
// Re-check the fill queue: fills may have arrived during this
|
|
1655
|
+
// tick's earlier phases (health check, dust cancels, divergence
|
|
1656
|
+
// corrections), after the pipeline gate above passed. Sizing a
|
|
1657
|
+
// correction from pre-fill budgets would under/over-fund the
|
|
1658
|
+
// repair — defer to the next tick so the fill cycle runs first
|
|
1659
|
+
// and side choice + sizing read fresh funds.
|
|
1660
|
+
const queuedFills = Array.isArray(bot?._incomingFillQueue)
|
|
1661
|
+
? bot._incomingFillQueue.length
|
|
1662
|
+
: 0;
|
|
1663
|
+
if (queuedFills > 0) {
|
|
1664
|
+
bot._log(`[SPREAD] Deferring spread check: ${queuedFills} fill(s) queued since pipeline gate; ` +
|
|
1665
|
+
`processing fills first for fresh funds`, 'debug');
|
|
1666
|
+
}
|
|
1667
|
+
else {
|
|
1668
|
+
const spreadResult = await bot.manager.checkSpreadCondition(BitShares, bot.updateOrdersOnChainPlan.bind(bot));
|
|
1669
|
+
if (await bot._abortFlowIfIllegalState(`${context} spread check`))
|
|
1670
|
+
return;
|
|
1671
|
+
if (spreadResult && spreadResult.ordersPlaced > 0) {
|
|
1672
|
+
bot._log(`✓ Spread correction during ${context}: ${spreadResult.ordersPlaced} order(s) placed`);
|
|
1673
|
+
await bot._persistAndRecoverIfNeeded();
|
|
1674
|
+
}
|
|
1541
1675
|
}
|
|
1542
1676
|
}
|
|
1543
1677
|
}
|
|
@@ -1652,7 +1786,7 @@ async function cancelDustOrders(bot, { buy: buyDust = [], sell: sellDust = [] }
|
|
|
1652
1786
|
}
|
|
1653
1787
|
if (syntheticFills.length === 0)
|
|
1654
1788
|
return { cancelledCount: 0, batchResult: null };
|
|
1655
|
-
const result = await bot._processFillsWithBatching(syntheticFills, new Set(), `dust cancel [${syntheticFills.map((o) => o.id).join(', ')}]
|
|
1789
|
+
const result = await bot._processFillsWithBatching(syntheticFills, new Set(), `dust cancel [${syntheticFills.map((o) => o.id).join(', ')}]`, { skipAnchorUpdate: true });
|
|
1656
1790
|
if (!result.aborted) {
|
|
1657
1791
|
await bot.manager.persistGrid();
|
|
1658
1792
|
}
|
|
@@ -1896,6 +2030,32 @@ function setupDustHealthCheckInterval(bot) {
|
|
|
1896
2030
|
bot._dustHealthCheckTimer.unref();
|
|
1897
2031
|
}
|
|
1898
2032
|
}
|
|
2033
|
+
/**
|
|
2034
|
+
* Retrigger the fill consumer once the recovery/pipeline window has fully
|
|
2035
|
+
* cleared. The consumer's defer branch (batchInFlight / recoverySyncInFlight /
|
|
2036
|
+
* broadcasting) returns WITHOUT rescheduling, so fills enqueued during a long
|
|
2037
|
+
* window would otherwise sit in the queue forever — which also keeps the
|
|
2038
|
+
* maintenance idle gate shut (queue-length check returns the full settle
|
|
2039
|
+
* delay) and blocks deferred grid resyncs and periodic maintenance.
|
|
2040
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
2041
|
+
*/
|
|
2042
|
+
function drainFillQueueAfterPipelineClear(bot) {
|
|
2043
|
+
try {
|
|
2044
|
+
if (!bot || bot._shuttingDown)
|
|
2045
|
+
return;
|
|
2046
|
+
if ((bot._recoverySyncInFlight || 0) > 0)
|
|
2047
|
+
return;
|
|
2048
|
+
if ((bot._batchInFlight || 0) > 0)
|
|
2049
|
+
return;
|
|
2050
|
+
if (bot.manager?.isBroadcastingActive?.())
|
|
2051
|
+
return;
|
|
2052
|
+
if (!bot._incomingFillQueue || bot._incomingFillQueue.length === 0)
|
|
2053
|
+
return;
|
|
2054
|
+
bot._log(`[FILL-QUEUE] Pipeline cleared; draining ${bot._incomingFillQueue.length} deferred fill(s).`, 'info');
|
|
2055
|
+
scheduleFillConsumerRestartFn(bot, chainOrders);
|
|
2056
|
+
}
|
|
2057
|
+
catch { }
|
|
2058
|
+
}
|
|
1899
2059
|
/**
|
|
1900
2060
|
* Request a full grid reset from fresh on-chain state.
|
|
1901
2061
|
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
@@ -1930,6 +2090,8 @@ async function requestGridReset(bot, reason = 'structural change', options = {})
|
|
|
1930
2090
|
}
|
|
1931
2091
|
finally {
|
|
1932
2092
|
bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
|
|
2093
|
+
if ((bot._recoverySyncInFlight || 0) === 0)
|
|
2094
|
+
drainFillQueueAfterPipelineClear(bot);
|
|
1933
2095
|
}
|
|
1934
2096
|
}
|
|
1935
2097
|
return bot.manager._fillProcessingLock.acquire(async () => performGridResync(bot, resetOptions));
|
|
@@ -1943,14 +2105,21 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
1943
2105
|
return;
|
|
1944
2106
|
bot.manager.requestStructuralGridResync = async (reason = 'structural recovery', details = {}) => {
|
|
1945
2107
|
if (bot._shuttingDown) {
|
|
2108
|
+
bot._warn(`[RECOVERY] Structural resync skip (shutting down): ${reason}`);
|
|
1946
2109
|
return { skipped: true, reason: 'shutting down' };
|
|
1947
2110
|
}
|
|
1948
2111
|
if (bot._structuralGridResyncRunning || bot._structuralGridResyncTimer) {
|
|
1949
|
-
|
|
2112
|
+
const why = bot._structuralGridResyncRunning ? 'already running' : 'already scheduled';
|
|
2113
|
+
bot._warn(`[RECOVERY] Structural resync skip (${why}): ${reason}`);
|
|
2114
|
+
return { skipped: true, reason: `structural grid resync ${why}` };
|
|
1950
2115
|
}
|
|
1951
2116
|
const unmatchedCount = Array.isArray(details?.unmatchedChainOrders)
|
|
1952
2117
|
? details.unmatchedChainOrders.length
|
|
1953
2118
|
: 0;
|
|
2119
|
+
// P5 max-defer: cap batchInFlight re-arm so a fill storm cannot starve
|
|
2120
|
+
// the resync forever (a fill-storm defer loop once kept the flag armed
|
|
2121
|
+
// indefinitely). After the cap, the resync forces instead of re-arming.
|
|
2122
|
+
const STRUCTURAL_RESYNC_MAX_DEFER_MS = 30000;
|
|
1954
2123
|
const runStructuralResync = async () => {
|
|
1955
2124
|
bot._structuralGridResyncTimer = null;
|
|
1956
2125
|
if (bot._shuttingDown)
|
|
@@ -1962,8 +2131,36 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
1962
2131
|
// completes; the fill consumer is already gated on _batchInFlight,
|
|
1963
2132
|
// so this only blocks the recovery itself, never new fills.
|
|
1964
2133
|
if (bot._batchInFlight > 0) {
|
|
1965
|
-
|
|
1966
|
-
|
|
2134
|
+
const now = Date.now();
|
|
2135
|
+
if (!bot._structuralGridResyncDeferStartedAt) {
|
|
2136
|
+
bot._structuralGridResyncDeferStartedAt = now;
|
|
2137
|
+
bot._structuralGridResyncDeferCount = 0;
|
|
2138
|
+
}
|
|
2139
|
+
bot._structuralGridResyncDeferCount = (bot._structuralGridResyncDeferCount || 0) + 1;
|
|
2140
|
+
const deferredMs = now - bot._structuralGridResyncDeferStartedAt;
|
|
2141
|
+
if (deferredMs >= STRUCTURAL_RESYNC_MAX_DEFER_MS) {
|
|
2142
|
+
bot._warn(`[RECOVERY] Structural resync max-defer reached (${deferredMs}ms, cap ${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms, deferred ${bot._structuralGridResyncDeferCount}x) — forcing despite _batchInFlight=${bot._batchInFlight}: ${reason}`);
|
|
2143
|
+
bot._structuralGridResyncDeferStartedAt = null;
|
|
2144
|
+
bot._structuralGridResyncDeferCount = 0;
|
|
2145
|
+
}
|
|
2146
|
+
else {
|
|
2147
|
+
// Fix #7 (docs/CONSOLIDATED_ORPHAN_FIX_SUMMARY.md §2): throttle the defer log to the
|
|
2148
|
+
// first occurrence per cap window (a fill storm can emit hundreds of
|
|
2149
|
+
// defer lines in seconds); count the rest and surface them only in
|
|
2150
|
+
// the final forced message.
|
|
2151
|
+
if (bot._structuralGridResyncDeferCount === 1) {
|
|
2152
|
+
bot._warn(`[RECOVERY] Structural resync defer (batchInFlight=${bot._batchInFlight}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`);
|
|
2153
|
+
}
|
|
2154
|
+
else {
|
|
2155
|
+
bot.manager?.logger?.log?.(`[RECOVERY] Structural resync defer (suppressed; count=${bot._structuralGridResyncDeferCount}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`, 'debug');
|
|
2156
|
+
}
|
|
2157
|
+
bot._structuralGridResyncTimer = setTimeout(runStructuralResync, TIMING.LOCK_REFRESH_MIN_MS);
|
|
2158
|
+
return;
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
2161
|
+
else {
|
|
2162
|
+
bot._structuralGridResyncDeferStartedAt = null;
|
|
2163
|
+
bot._structuralGridResyncDeferCount = 0;
|
|
1967
2164
|
}
|
|
1968
2165
|
bot._structuralGridResyncRunning++;
|
|
1969
2166
|
try {
|
|
@@ -1989,14 +2186,30 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
1989
2186
|
}
|
|
1990
2187
|
const suffix = unmatchedCount > 0 ? ` (${unmatchedCount} unmatched chain order(s))` : '';
|
|
1991
2188
|
bot._warn(`[RECOVERY] Running structural full grid resync for ${reason}${suffix}`);
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2189
|
+
// Same protection as the persisted-grid reload above: the
|
|
2190
|
+
// reset's reconcile Phase-2 placement runs for minutes; a
|
|
2191
|
+
// fill arriving mid-reset must not start a COW batch whose
|
|
2192
|
+
// broadcast overlaps it (commit refusal + duplicate
|
|
2193
|
+
// generation orphans). _recoverFromPersistedGrid already
|
|
2194
|
+
// incremented and released the counter; increment again
|
|
2195
|
+
// for the full-resync branch.
|
|
2196
|
+
bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
|
|
2197
|
+
let resetResult;
|
|
2198
|
+
try {
|
|
2199
|
+
resetResult = await bot.requestGridReset('rms_structural_grid_resync', {
|
|
2200
|
+
refreshCenterPrice: false,
|
|
2201
|
+
// Structural resync is a chain-read/rebuild; do not let a
|
|
2202
|
+
// fill storm starve it behind the idle cooldown or the
|
|
2203
|
+
// fill-processing lock.
|
|
2204
|
+
skipIdle: true,
|
|
2205
|
+
skipFillLock: true,
|
|
2206
|
+
});
|
|
2207
|
+
}
|
|
2208
|
+
finally {
|
|
2209
|
+
bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
|
|
2210
|
+
if ((bot._recoverySyncInFlight || 0) === 0)
|
|
2211
|
+
drainFillQueueAfterPipelineClear(bot);
|
|
2212
|
+
}
|
|
2000
2213
|
if (resetResult && bot.manager?._recoveryState) {
|
|
2001
2214
|
bot.manager._recoveryState = { ...bot.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
|
|
2002
2215
|
}
|
|
@@ -2101,7 +2314,7 @@ async function syncOpenOrdersAndProcessFillsImpl(bot, tag) {
|
|
|
2101
2314
|
if (syncResult?.filledOrders?.length > 0) {
|
|
2102
2315
|
bot._refreshDynamicWeightDistribution(`${tag} sync-fill`);
|
|
2103
2316
|
bot._log(`[SYNC-CHAIN] ${syncResult.filledOrders.length} filled order(s) found during ${tag}`, 'info');
|
|
2104
|
-
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), `${tag} sync-fill
|
|
2317
|
+
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), `${tag} sync-fill`, { isReplay: true });
|
|
2105
2318
|
if (!batchResult?.aborted) {
|
|
2106
2319
|
// Reassign the returned snapshot to the post-fill re-read: the
|
|
2107
2320
|
// caller feeds openOrders into reconcileGridOrders, which must
|
|
@@ -2128,9 +2341,10 @@ async function syncOpenOrdersAndProcessFillsImpl(bot, tag) {
|
|
|
2128
2341
|
return { syncResult: null, aborted: true, hasUnmatched: -1, openOrders: null };
|
|
2129
2342
|
}
|
|
2130
2343
|
}
|
|
2131
|
-
export { loadBotsConfigSnapshot, refreshDynamicWeightDistribution, performGridResync, updateBotGridResetMetadata, handlePendingTriggerReset, setupTriggerFileDetection, performPeriodicGridChecks, isOpenOrdersSyncLoopEnabled, startOpenOrdersSyncLoop, stopOpenOrdersSyncLoop, setupBlockchainFetchInterval, stopBlockchainFetchInterval, executeMaintenanceLogic, cancelDustOrders, isOrderDoesNotExistError, runGridMaintenance, stopMarketAdapterPm2, releaseMarketAdapterRuntime, syncMarketAdapterOnPeriodicConfigCheck, findSnapshotBotForRuntimeConfig, runtimeConfigNeedsMarketAdapter, usesAmaGridPrice, checkBtsBalanceAndAcquire, acquireBts, runDustHealthCheck, setupDustHealthCheckInterval, requestGridReset, wireStructuralGridResyncRequest, getPipelineSignals, markGridActivity, getMetrics, syncOpenOrdersAndProcessFills };
|
|
2344
|
+
export { loadBotsConfigSnapshot, isWrapperAdapterOwner, refreshDynamicWeightDistribution, performGridResync, updateBotGridResetMetadata, handlePendingTriggerReset, setupTriggerFileDetection, performPeriodicGridChecks, isOpenOrdersSyncLoopEnabled, startOpenOrdersSyncLoop, stopOpenOrdersSyncLoop, setupBlockchainFetchInterval, stopBlockchainFetchInterval, setupBotsConfigPollInterval, stopBotsConfigPollInterval, executeMaintenanceLogic, cancelDustOrders, isOrderDoesNotExistError, runGridMaintenance, stopMarketAdapterPm2, releaseMarketAdapterRuntime, syncMarketAdapterOnPeriodicConfigCheck, findSnapshotBotForRuntimeConfig, runtimeConfigNeedsMarketAdapter, usesAmaGridPrice, checkBtsBalanceAndAcquire, acquireBts, runDustHealthCheck, setupDustHealthCheckInterval, requestGridReset, wireStructuralGridResyncRequest, getPipelineSignals, markGridActivity, getMetrics, syncOpenOrdersAndProcessFills };
|
|
2132
2345
|
export default {
|
|
2133
2346
|
loadBotsConfigSnapshot,
|
|
2347
|
+
isWrapperAdapterOwner,
|
|
2134
2348
|
refreshDynamicWeightDistribution,
|
|
2135
2349
|
performGridResync,
|
|
2136
2350
|
updateBotGridResetMetadata,
|
|
@@ -2142,6 +2356,8 @@ export default {
|
|
|
2142
2356
|
stopOpenOrdersSyncLoop,
|
|
2143
2357
|
setupBlockchainFetchInterval,
|
|
2144
2358
|
stopBlockchainFetchInterval,
|
|
2359
|
+
setupBotsConfigPollInterval,
|
|
2360
|
+
stopBotsConfigPollInterval,
|
|
2145
2361
|
executeMaintenanceLogic,
|
|
2146
2362
|
cancelDustOrders,
|
|
2147
2363
|
isOrderDoesNotExistError,
|