dexbot 1.4.6 → 1.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/credential-daemon.d.ts.map +1 -1
- package/dist/credential-daemon.js +416 -92
- package/dist/credential-daemon.js.map +1 -1
- package/dist/dexbot.d.ts.map +1 -1
- package/dist/dexbot.js +61 -27
- package/dist/dexbot.js.map +1 -1
- package/dist/modules/account_bots.d.ts.map +1 -1
- package/dist/modules/account_bots.js +14 -24
- package/dist/modules/account_bots.js.map +1 -1
- package/dist/modules/bitshares-native/transport.d.ts.map +1 -1
- package/dist/modules/bitshares-native/transport.js +21 -0
- package/dist/modules/bitshares-native/transport.js.map +1 -1
- package/dist/modules/broadcast_failure.d.ts +21 -0
- package/dist/modules/broadcast_failure.d.ts.map +1 -0
- package/dist/modules/broadcast_failure.js +95 -0
- package/dist/modules/broadcast_failure.js.map +1 -0
- package/dist/modules/chain_keys.d.ts +2 -2
- package/dist/modules/chain_keys.d.ts.map +1 -1
- package/dist/modules/chain_keys.js +30 -56
- package/dist/modules/chain_keys.js.map +1 -1
- package/dist/modules/chain_orders.d.ts +79 -1
- package/dist/modules/chain_orders.d.ts.map +1 -1
- package/dist/modules/chain_orders.js +154 -10
- package/dist/modules/chain_orders.js.map +1 -1
- package/dist/modules/constants.d.ts +1 -0
- package/dist/modules/constants.d.ts.map +1 -1
- package/dist/modules/constants.js +21 -7
- package/dist/modules/constants.js.map +1 -1
- package/dist/modules/daemon_node_health.d.ts +33 -0
- package/dist/modules/daemon_node_health.d.ts.map +1 -0
- package/dist/modules/daemon_node_health.js +120 -0
- package/dist/modules/daemon_node_health.js.map +1 -0
- package/dist/modules/dexbot_class.d.ts +12 -3
- package/dist/modules/dexbot_class.d.ts.map +1 -1
- package/dist/modules/dexbot_class.js +19 -5
- package/dist/modules/dexbot_class.js.map +1 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +46 -1
- package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_cow_runtime.js +754 -64
- package/dist/modules/dexbot_cow_runtime.js.map +1 -1
- package/dist/modules/dexbot_credential_client.d.ts +15 -0
- package/dist/modules/dexbot_credential_client.d.ts.map +1 -1
- package/dist/modules/dexbot_credential_client.js +114 -141
- package/dist/modules/dexbot_credential_client.js.map +1 -1
- package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_fill_runtime.js +45 -15
- package/dist/modules/dexbot_fill_runtime.js.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts +5 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +238 -85
- package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
- package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_startup_runtime.js +95 -33
- package/dist/modules/dexbot_startup_runtime.js.map +1 -1
- package/dist/modules/dexbot_state_recovery.d.ts.map +1 -1
- package/dist/modules/dexbot_state_recovery.js +24 -2
- package/dist/modules/dexbot_state_recovery.js.map +1 -1
- package/dist/modules/key_store.d.ts.map +1 -1
- package/dist/modules/key_store.js +5 -4
- package/dist/modules/key_store.js.map +1 -1
- package/dist/modules/launcher/credential_daemon.d.ts.map +1 -1
- package/dist/modules/launcher/credential_daemon.js +15 -3
- package/dist/modules/launcher/credential_daemon.js.map +1 -1
- package/dist/modules/node_failure_ledger.d.ts +39 -0
- package/dist/modules/node_failure_ledger.d.ts.map +1 -0
- package/dist/modules/node_failure_ledger.js +88 -0
- package/dist/modules/node_failure_ledger.js.map +1 -0
- package/dist/modules/node_health_cache.d.ts +2 -1
- package/dist/modules/node_health_cache.d.ts.map +1 -1
- package/dist/modules/node_health_cache.js +25 -0
- package/dist/modules/node_health_cache.js.map +1 -1
- package/dist/modules/node_manager.d.ts +3 -0
- package/dist/modules/node_manager.d.ts.map +1 -1
- package/dist/modules/node_manager.js +41 -10
- package/dist/modules/node_manager.js.map +1 -1
- package/dist/modules/order/accounting.d.ts +26 -20
- package/dist/modules/order/accounting.d.ts.map +1 -1
- package/dist/modules/order/accounting.js +281 -139
- package/dist/modules/order/accounting.js.map +1 -1
- package/dist/modules/order/grid.d.ts.map +1 -1
- package/dist/modules/order/grid.js +66 -14
- package/dist/modules/order/grid.js.map +1 -1
- package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile.js +136 -56
- package/dist/modules/order/grid_reconcile.js.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts +6 -5
- package/dist/modules/order/grid_reconcile_internal.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.js +388 -61
- package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
- package/dist/modules/order/manager.d.ts +98 -4
- package/dist/modules/order/manager.d.ts.map +1 -1
- package/dist/modules/order/manager.js +379 -92
- package/dist/modules/order/manager.js.map +1 -1
- package/dist/modules/order/strategy.d.ts.map +1 -1
- package/dist/modules/order/strategy.js +24 -1
- package/dist/modules/order/strategy.js.map +1 -1
- package/dist/modules/order/sync_engine.d.ts +16 -3
- package/dist/modules/order/sync_engine.d.ts.map +1 -1
- package/dist/modules/order/sync_engine.js +341 -253
- package/dist/modules/order/sync_engine.js.map +1 -1
- package/dist/modules/order/utils/math.d.ts +79 -1
- package/dist/modules/order/utils/math.d.ts.map +1 -1
- package/dist/modules/order/utils/math.js +18 -0
- package/dist/modules/order/utils/math.js.map +1 -1
- package/dist/modules/order/utils/order.d.ts +44 -1
- package/dist/modules/order/utils/order.d.ts.map +1 -1
- package/dist/modules/order/utils/order.js +91 -5
- package/dist/modules/order/utils/order.js.map +1 -1
- package/dist/modules/order/utils/system.d.ts +5 -1
- package/dist/modules/order/utils/system.d.ts.map +1 -1
- package/dist/modules/order/utils/system.js +7 -5
- package/dist/modules/order/utils/system.js.map +1 -1
- package/dist/modules/order/utils/withPoolRef.d.ts.map +1 -1
- package/dist/modules/order/utils/withPoolRef.js +39 -4
- package/dist/modules/order/utils/withPoolRef.js.map +1 -1
- package/dist/modules/socket_json_client.d.ts +14 -0
- package/dist/modules/socket_json_client.d.ts.map +1 -0
- package/dist/modules/socket_json_client.js +100 -0
- package/dist/modules/socket_json_client.js.map +1 -0
- package/package.json +2 -1
- package/scripts/clear-all.sh +4 -4
- package/scripts/clear-logs.sh +4 -4
|
@@ -114,6 +114,7 @@ declare function updateBotGridResetMetadata(botKey: any, options?: {
|
|
|
114
114
|
* recalculate the grid, persist, and record reset metadata.
|
|
115
115
|
* @param {import('./dexbot_class').DEXBot} bot
|
|
116
116
|
* @param {import('./types').GridResyncOptions} [options] - Grid resync options
|
|
117
|
+
* @param {boolean} [options.skipIdle=false] - Skip the idle-cooldown deferral
|
|
117
118
|
* @returns {Promise<boolean>} True if resync succeeded
|
|
118
119
|
*/
|
|
119
120
|
declare function performGridResync(bot: any, options?: {
|
|
@@ -121,6 +122,7 @@ declare function performGridResync(bot: any, options?: {
|
|
|
121
122
|
centerRefreshContext?: string;
|
|
122
123
|
centerRefreshLabel?: string;
|
|
123
124
|
resetSource?: string;
|
|
125
|
+
skipIdle?: boolean;
|
|
124
126
|
}): Promise<boolean>;
|
|
125
127
|
/**
|
|
126
128
|
* Handle a pending trigger file detected at startup or during runtime.
|
|
@@ -259,11 +261,13 @@ declare function setupDustHealthCheckInterval(bot: any): void;
|
|
|
259
261
|
* Request a full grid reset from fresh on-chain state.
|
|
260
262
|
* @param {import('./dexbot_class').DEXBot} bot
|
|
261
263
|
* @param {string} [reason='structural change']
|
|
262
|
-
* @param {{refreshCenterPrice?: boolean}} [options={}]
|
|
264
|
+
* @param {{refreshCenterPrice?: boolean, skipIdle?: boolean, skipFillLock?: boolean}} [options={}]
|
|
263
265
|
* @returns {Promise<Object>}
|
|
264
266
|
*/
|
|
265
267
|
declare function requestGridReset(bot: any, reason?: any, options?: {
|
|
266
268
|
refreshCenterPrice?: boolean;
|
|
269
|
+
skipIdle?: boolean;
|
|
270
|
+
skipFillLock?: boolean;
|
|
267
271
|
}): Promise<any>;
|
|
268
272
|
/**
|
|
269
273
|
* Wire the structural grid resync request handler on the manager.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dexbot_maintenance_runtime.d.ts","sourceRoot":"","sources":["../../modules/dexbot_maintenance_runtime.ts"],"names":[],"mappings":"AA6FA;;;;GAIG;AACH,iBAAS,gBAAgB,CAAC,GAAG,EAAE,GAAG,WAGjC;AAED;;;;;;GAMG;AACH,iBAAS,+BAA+B,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,OAalE;AAED;;;;;GAKG;AACH,iBAAS,+BAA+B,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,WAMlE;AA6HD;;;GAGG;AACH,iBAAS,sBAAsB;;;;;;;;;;;;EAgC9B;AAuGD;;;GAGG;AACH,iBAAe,oBAAoB,kBAElC;AAED;;;;;;GAMG;AACH,iBAAe,sCAAsC,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,GAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4IxF;AAED;;;;;;GAMG;AACH,iBAAS,gCAAgC,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,GAAe;;;;;;;;;;EA+D3E;AAyHD;;;;;;;GAOG;AACH,iBAAS,0BAA0B,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,OA0BxG;AAED
|
|
1
|
+
{"version":3,"file":"dexbot_maintenance_runtime.d.ts","sourceRoot":"","sources":["../../modules/dexbot_maintenance_runtime.ts"],"names":[],"mappings":"AA6FA;;;;GAIG;AACH,iBAAS,gBAAgB,CAAC,GAAG,EAAE,GAAG,WAGjC;AAED;;;;;;GAMG;AACH,iBAAS,+BAA+B,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,OAalE;AAED;;;;;GAKG;AACH,iBAAS,+BAA+B,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,WAMlE;AA6HD;;;GAGG;AACH,iBAAS,sBAAsB;;;;;;;;;;;;EAgC9B;AAuGD;;;GAGG;AACH,iBAAe,oBAAoB,kBAElC;AAED;;;;;;GAMG;AACH,iBAAe,sCAAsC,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,GAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4IxF;AAED;;;;;;GAMG;AACH,iBAAS,gCAAgC,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,GAAe;;;;;;;;;;EA+D3E;AAyHD;;;;;;;GAOG;AACH,iBAAS,0BAA0B,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,OA0BxG;AAED;;;;;;;GAOG;AACH,iBAAS,iBAAiB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE;IAC1C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACjB,oBA8IL;AAED;;;;;GAKG;AACH,iBAAe,yBAAyB,CAAC,GAAG,EAAE,GAAG,oBAkBhD;AAED;;;;;GAKG;AACH,iBAAe,yBAAyB,CAAC,GAAG,EAAE,GAAG,iBAiDhD;AAED;;;;;GAKG;AACH,iBAAe,yBAAyB,CAAC,GAAG,EAAE,GAAG,iBAMhD;AAED;;;;GAIG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,WAK5C;AAED;;;;GAIG;AACH,iBAAS,uBAAuB,CAAC,GAAG,EAAE,GAAG,QA2ExC;AAED;;;;GAIG;AACH,iBAAe,sBAAsB,CAAC,GAAG,EAAE,GAAG,iBAK7C;AAED;;;;GAIG;AACH,iBAAS,4BAA4B,CAAC,GAAG,EAAE,GAAG,QAkI7C;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,QAM5C;AAED;;;;;;;GAOG;AACH,iBAAe,2BAA2B,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,GAAE,GAAgB,gBAiB1F;AAID;;;;;GAKG;AACH,iBAAS,wBAAwB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,WAY3D;AAiGD;;;;;;;GAOG;AACH,iBAAe,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,iBAoQ5D;AA+BD;;;;;;;;;;GAUG;AACH,iBAAe,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAY,EAAE,IAAI,EAAE,QAAa,EAAE,GAAE,GAAQ;;;;;;;;GAwD7F;AAED;;;;;;;;GAQG;AACH,iBAAe,kBAAkB,CAC7B,GAAG,EAAE,GAAG,EACR,OAAO,GAAE,GAAgB,EACzB,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,iBAmCvC;AAID;;;;;;GAMG;AACH,iBAAe,yBAAyB,CAAC,GAAG,EAAE,GAAG,iBAoDhD;AAED;;;;;;GAMG;AACH,iBAAe,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,iBAmF/C;AAED;;;GAGG;AACH,iBAAe,kBAAkB,CAAC,GAAG,EAAE,GAAG,iBAkCzC;AAED;;;GAGG;AACH,iBAAS,4BAA4B,CAAC,GAAG,EAAE,GAAG,QAO7C;AAED;;;;;;GAMG;AACH,iBAAe,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,GAAE,GAAyB,EAAE,OAAO,GAAE;IAAE,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,CAAC,EAAE,OAAO,CAAA;CAAO,gBA6BtK;AAED;;;GAGG;AACH,iBAAS,+BAA+B,CAAC,GAAG,EAAE,GAAG,QA8EhD;AAED;;;;GAIG;AACH,iBAAS,kBAAkB,CAAC,GAAG,EAAE,GAAG;;;;;;EASnC;AAED;;;;GAIG;AACH,iBAAS,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,GAAE,GAAgB,QAG3D;AAED;;;;GAIG;AACH,iBAAS,UAAU,CAAC,GAAG,EAAE,GAAG,OAW3B;AAED;;;;;GAKG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;;;;;GAsD9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAED,kBAiCE"}
|
|
@@ -5,6 +5,7 @@ const fs = require('fs');
|
|
|
5
5
|
const { path } = require('./path_api');
|
|
6
6
|
const { spawn } = require('child_process');
|
|
7
7
|
const chainOrders = require('./chain_orders');
|
|
8
|
+
const { readOpenOrdersGuarded } = require('./chain_orders');
|
|
8
9
|
const grid = require('./order/grid');
|
|
9
10
|
const { ORDER_STATES, ORDER_TYPES, TIMING, BTS_PRECISION, NATIVE_CLIENT } = require('./constants');
|
|
10
11
|
const { PATHS } = require('./paths');
|
|
@@ -15,7 +16,6 @@ const fundRegistry = require('./fund_registry');
|
|
|
15
16
|
const { BitShares } = require('./bitshares_client');
|
|
16
17
|
const { BroadcastUncertainError } = require('./dexbot_credential_client');
|
|
17
18
|
const { Config } = require('./config');
|
|
18
|
-
function getNodeManager(...args) { return require('./bitshares_client').getNodeManager(...args); }
|
|
19
19
|
function hasOpenOrdersSyncLoopMsSet(...args) { return require('./config').hasOpenOrdersSyncLoopMsSet(...args); }
|
|
20
20
|
function getOpenOrdersSyncLoopMs(...args) { return require('./config').getOpenOrdersSyncLoopMs(...args); }
|
|
21
21
|
function isGridBloated(...args) { return grid.isGridBloated(...args); }
|
|
@@ -706,6 +706,7 @@ function updateBotGridResetMetadata(botKey, options = {}) {
|
|
|
706
706
|
* recalculate the grid, persist, and record reset metadata.
|
|
707
707
|
* @param {import('./dexbot_class').DEXBot} bot
|
|
708
708
|
* @param {import('./types').GridResyncOptions} [options] - Grid resync options
|
|
709
|
+
* @param {boolean} [options.skipIdle=false] - Skip the idle-cooldown deferral
|
|
709
710
|
* @returns {Promise<boolean>} True if resync succeeded
|
|
710
711
|
*/
|
|
711
712
|
function performGridResync(bot, options = {}) {
|
|
@@ -715,8 +716,9 @@ function performGridResync(bot, options = {}) {
|
|
|
715
716
|
const centerRefreshContext = options.centerRefreshContext || (refreshCenterPrice ? 'grid reset recenter' : 'grid resync');
|
|
716
717
|
const centerRefreshLabel = options.centerRefreshLabel || (refreshCenterPrice ? 'grid reset' : 'grid resync');
|
|
717
718
|
const resetSource = options.resetSource || (refreshCenterPrice ? 'manual_grid_resync' : 'dexbot_grid_resync');
|
|
719
|
+
const skipIdle = options.skipIdle === true;
|
|
718
720
|
const idleDelayMs = getMaintenanceIdleDelayMs(self);
|
|
719
|
-
if (idleDelayMs > 0) {
|
|
721
|
+
if (!skipIdle && idleDelayMs > 0) {
|
|
720
722
|
self._log(`[MAINT-IDLE] Deferring grid resync until bot is idle` +
|
|
721
723
|
` (next check in ${Math.ceil(idleDelayMs / TIMING.MILLISECONDS_PER_SECOND)}s)`, 'info');
|
|
722
724
|
scheduleDeferredGridResync(self, options);
|
|
@@ -767,7 +769,22 @@ function performGridResync(bot, options = {}) {
|
|
|
767
769
|
// fresh weights so cancelDustOrders uses live distribution (Issue M).
|
|
768
770
|
refreshDynamicWeightDistribution(self, 'grid resync');
|
|
769
771
|
}
|
|
770
|
-
|
|
772
|
+
// Truncated-read guard: recalculateGrid's readOpenOrdersFn drives
|
|
773
|
+
// syncFromOpenOrders absence decisions (pass-1 phantom cleanup).
|
|
774
|
+
// A partial get_full_accounts window must abort the resync instead
|
|
775
|
+
// of virtualizing live ACTIVE slots; the trigger file is retained
|
|
776
|
+
// so the resync retries on a clean read.
|
|
777
|
+
let resyncReadAmbiguous = false;
|
|
778
|
+
const readFn = async () => {
|
|
779
|
+
const orders = await readOpenOrdersGuarded(chainOrders, self.accountId, {
|
|
780
|
+
log: (message, level) => self._log(message, level),
|
|
781
|
+
label: 'GRID-RESYNC',
|
|
782
|
+
detail: 'trigger-file resync',
|
|
783
|
+
});
|
|
784
|
+
if (orders === null)
|
|
785
|
+
resyncReadAmbiguous = true;
|
|
786
|
+
return orders;
|
|
787
|
+
};
|
|
771
788
|
await recalculateGrid(self.manager, {
|
|
772
789
|
readOpenOrdersFn: readFn,
|
|
773
790
|
chainOrders,
|
|
@@ -775,7 +792,13 @@ function performGridResync(bot, options = {}) {
|
|
|
775
792
|
privateKey: self.privateKey,
|
|
776
793
|
config: self.config,
|
|
777
794
|
});
|
|
778
|
-
|
|
795
|
+
if (resyncReadAmbiguous) {
|
|
796
|
+
self._warn('[GRID-RESYNC] Aborted trigger-file resync: open-order read ambiguous (truncated); retaining trigger file for retry.');
|
|
797
|
+
return false;
|
|
798
|
+
}
|
|
799
|
+
await self.manager._fundLock.acquire(async () => {
|
|
800
|
+
self.manager.funds.btsFeesOwed = 0;
|
|
801
|
+
});
|
|
779
802
|
await self.manager.persistGrid();
|
|
780
803
|
success = true;
|
|
781
804
|
if (updateBotGridResetMetadata(self.config?.botKey, {
|
|
@@ -942,22 +965,39 @@ function startOpenOrdersSyncLoop(bot) {
|
|
|
942
965
|
}
|
|
943
966
|
bot._mainLoopActive = true;
|
|
944
967
|
bot._log(`Open-orders sync loop started (every ${loopDelayMs}ms, dryRun=${!!bot.config.dryRun})`);
|
|
945
|
-
const readOpenOrdersFn = chainOrders.readOpenOrders;
|
|
946
968
|
bot._mainLoopPromise = (async () => {
|
|
947
969
|
while (bot._mainLoopActive && !bot._shuttingDown) {
|
|
948
970
|
try {
|
|
949
971
|
if (bot.manager && bot.accountId && !bot.config.dryRun) {
|
|
950
|
-
|
|
972
|
+
// Skip while a structural recovery reload is in flight:
|
|
973
|
+
// _recoverFromPersistedGrid runs WITHOUT the fill lock, so
|
|
974
|
+
// the lock gate alone would let this loop synchronizeWithChain
|
|
975
|
+
// (and even fire a rebalance broadcast) into a half-rebuilt
|
|
976
|
+
// grid. Same isolation the fill consumer gets via
|
|
977
|
+
// _recoverySyncInFlight.
|
|
978
|
+
if (!bot._recoverySyncInFlight &&
|
|
979
|
+
!bot.manager._fillProcessingLock.isLocked() &&
|
|
951
980
|
bot.manager._fillProcessingLock.getQueueLength() === 0) {
|
|
952
981
|
await bot.manager._fillProcessingLock.acquire(async () => {
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
982
|
+
// Truncated-read guard: syncing on a partial
|
|
983
|
+
// get_full_accounts window would virtualize live
|
|
984
|
+
// ACTIVE slots (pass-1 phantom cleanup) and
|
|
985
|
+
// re-create them as duplicates. Defer to a clean
|
|
986
|
+
// read — fill subscription events keep the bot
|
|
987
|
+
// responsive in the meantime.
|
|
988
|
+
const chainOpenOrders = await readOpenOrdersGuarded(chainOrders, bot.accountId, {
|
|
989
|
+
log: (message, level) => bot._log(message, level),
|
|
990
|
+
label: 'OPEN-ORDERS-SYNC',
|
|
991
|
+
});
|
|
992
|
+
if (chainOpenOrders !== null) {
|
|
993
|
+
const syncResult = await bot.manager.synchronizeWithChain(chainOpenOrders, 'readOpenOrders');
|
|
994
|
+
if (syncResult?.filledOrders && syncResult.filledOrders.length > 0) {
|
|
995
|
+
bot._log(`Open-orders sync loop: ${syncResult.filledOrders.length} grid order(s) found filled on-chain. Triggering rebalance.`, 'info');
|
|
996
|
+
bot._markGridActivity?.('open-orders sync fill');
|
|
997
|
+
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), 'open-orders sync fill rebalance');
|
|
998
|
+
if (!batchResult?.aborted) {
|
|
999
|
+
await bot.manager.persistGrid();
|
|
1000
|
+
}
|
|
961
1001
|
}
|
|
962
1002
|
}
|
|
963
1003
|
// Run grid health / dust detection after every sync tick so
|
|
@@ -1068,23 +1108,34 @@ function setupBlockchainFetchInterval(bot) {
|
|
|
1068
1108
|
let chainOpenOrders = [];
|
|
1069
1109
|
if (!bot.config.dryRun) {
|
|
1070
1110
|
try {
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1111
|
+
// Truncated-read guard: a partial get_full_accounts
|
|
1112
|
+
// window would make synchronizeWithChain's pass-1
|
|
1113
|
+
// virtualize live ACTIVE slots that are simply missing
|
|
1114
|
+
// from the window (then re-create them as duplicates).
|
|
1115
|
+
// Defer the sync to a clean read; fills are still
|
|
1116
|
+
// caught by subscription events and the next cycle.
|
|
1117
|
+
chainOpenOrders = await readOpenOrdersGuarded(chainOrders, bot.accountId, {
|
|
1118
|
+
log: (message, level) => bot._log(message, level),
|
|
1119
|
+
label: 'PERIODIC-SYNC',
|
|
1120
|
+
});
|
|
1121
|
+
if (chainOpenOrders !== null) {
|
|
1122
|
+
const syncResult = await bot.manager.synchronizeWithChain(chainOpenOrders, 'periodicBlockchainFetch');
|
|
1123
|
+
if (syncResult.filledOrders && syncResult.filledOrders.length > 0) {
|
|
1124
|
+
bot._log(`Periodic sync: ${syncResult.filledOrders.length} grid order(s) found filled on-chain. Triggering rebalance.`, 'info');
|
|
1125
|
+
bot._markGridActivity?.('periodic sync fill rebalance');
|
|
1126
|
+
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), 'periodic sync fill rebalance');
|
|
1127
|
+
if (!batchResult?.aborted) {
|
|
1128
|
+
await bot.manager.persistGrid();
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
if (syncResult.unmatchedChainOrders && syncResult.unmatchedChainOrders.length > 0) {
|
|
1132
|
+
const sample = syncResult.unmatchedChainOrders
|
|
1133
|
+
.slice(0, 3)
|
|
1134
|
+
.map(formatUnmatchedChainOrder)
|
|
1135
|
+
.join(' | ');
|
|
1136
|
+
bot._log(`Periodic sync: ${syncResult.unmatchedChainOrders.length} chain order(s) not in grid ` +
|
|
1137
|
+
`(surplus/divergence)${sample ? `: ${sample}` : ''}`, 'warn');
|
|
1079
1138
|
}
|
|
1080
|
-
}
|
|
1081
|
-
if (syncResult.unmatchedChainOrders && syncResult.unmatchedChainOrders.length > 0) {
|
|
1082
|
-
const sample = syncResult.unmatchedChainOrders
|
|
1083
|
-
.slice(0, 3)
|
|
1084
|
-
.map(formatUnmatchedChainOrder)
|
|
1085
|
-
.join(' | ');
|
|
1086
|
-
bot._log(`Periodic sync: ${syncResult.unmatchedChainOrders.length} chain order(s) not in grid ` +
|
|
1087
|
-
`(surplus/divergence)${sample ? `: ${sample}` : ''}`, 'warn');
|
|
1088
1139
|
}
|
|
1089
1140
|
}
|
|
1090
1141
|
catch (err) {
|
|
@@ -1311,30 +1362,43 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
1311
1362
|
// LIGHTWEIGHT_SYNC_CHECK_INTERVAL_MS to limit blockchain query load.
|
|
1312
1363
|
// Skip if the COW pipeline has active CREATEs in flight to avoid racing
|
|
1313
1364
|
// with batch broadcasts — transient mismatches are expected during a COW cycle.
|
|
1314
|
-
|
|
1365
|
+
// Also skip while a structural recovery reload is in flight (it runs without
|
|
1366
|
+
// the fill lock, so this check must gate on the recovery flag explicitly).
|
|
1367
|
+
if (!bot._recoverySyncInFlight &&
|
|
1368
|
+
!bot._batchInFlight &&
|
|
1315
1369
|
!(bot.manager?._pendingBroadcasts?.size > 0) &&
|
|
1316
1370
|
(bot._lightweightSyncCheckAt == null || Date.now() - bot._lightweightSyncCheckAt >= TIMING.LIGHTWEIGHT_SYNC_CHECK_INTERVAL_MS)) {
|
|
1317
1371
|
bot._lightweightSyncCheckAt = Date.now();
|
|
1318
1372
|
try {
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
await bot.manager.synchronizeWithChain(chainOpenOrdersResult, 'readOpenOrders');
|
|
1333
|
-
}
|
|
1373
|
+
// Truncated-read guard: a partial window undercounts the chain
|
|
1374
|
+
// order count, which would fabricate a divergence and trigger a
|
|
1375
|
+
// full synchronizeWithChain on a partial snapshot (pass-1 phantom
|
|
1376
|
+
// virtualization). Defer to a clean read.
|
|
1377
|
+
const chainOpenOrdersResult = await readOpenOrdersGuarded(chainOrders, bot.accountId, {
|
|
1378
|
+
log: (message, level) => bot._log(message, level),
|
|
1379
|
+
label: 'LIGHTWEIGHT-SYNC',
|
|
1380
|
+
skipMessage: (kind) => `[LIGHTWEIGHT-SYNC] Open-order read ${kind}; skipping consistency check (partial snapshot cannot drive count comparisons)`,
|
|
1381
|
+
});
|
|
1382
|
+
if (chainOpenOrdersResult !== null) {
|
|
1383
|
+
const assets = bot.manager?.assets;
|
|
1384
|
+
if (!assets) {
|
|
1385
|
+
bot._log('[LIGHTWEIGHT-SYNC] Skipped: manager assets not available', 'debug');
|
|
1334
1386
|
}
|
|
1335
|
-
else
|
|
1336
|
-
|
|
1337
|
-
|
|
1387
|
+
else {
|
|
1388
|
+
const chainOrdersCount = chainOpenOrdersResult.filter((o) => parseChainOrder(o, assets) !== null).length;
|
|
1389
|
+
const gridActive = Array.from(bot.manager.orders.values()).filter((o) => isOrderOnChain(o)).length;
|
|
1390
|
+
const diff = Math.abs(chainOrdersCount - gridActive);
|
|
1391
|
+
if (diff > 2) {
|
|
1392
|
+
bot._log(`[LIGHTWEIGHT-SYNC] Order count mismatch: chain=${chainOrdersCount}, grid=${gridActive} ` +
|
|
1393
|
+
`(diff=${diff}). Triggering targeted sync to reconcile.`, 'warn');
|
|
1394
|
+
if (bot.manager?.synchronizeWithChain) {
|
|
1395
|
+
await bot.manager.synchronizeWithChain(chainOpenOrdersResult, 'readOpenOrders');
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
else if (diff > 0) {
|
|
1399
|
+
bot._log(`[LIGHTWEIGHT-SYNC] Order count mismatch: chain=${chainOrdersCount}, grid=${gridActive} ` +
|
|
1400
|
+
`(diff=${diff}). Minor divergence — expected during normal operation.`, 'debug');
|
|
1401
|
+
}
|
|
1338
1402
|
}
|
|
1339
1403
|
}
|
|
1340
1404
|
}
|
|
@@ -1361,13 +1425,23 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
1361
1425
|
return;
|
|
1362
1426
|
const pipelineStatus = bot.manager.isPipelineEmpty(bot._getPipelineSignals());
|
|
1363
1427
|
if (pipelineStatus.isEmpty) {
|
|
1364
|
-
const
|
|
1365
|
-
if (
|
|
1428
|
+
const repairedFromTarget = await maybeRunTargetedDriftReconciliation(bot, context);
|
|
1429
|
+
if (repairedFromTarget) {
|
|
1366
1430
|
const freshHealth = await bot.manager.checkGridHealth(bot.updateOrdersOnChainPlan.bind(bot));
|
|
1367
1431
|
if (await bot._abortFlowIfIllegalState(`${context} post-reconcile health check`))
|
|
1368
1432
|
return;
|
|
1369
1433
|
healthResult = freshHealth;
|
|
1370
1434
|
}
|
|
1435
|
+
if (!repairedFromTarget) {
|
|
1436
|
+
const autoCancelResult = await bot._autoCancelOneUnmatchedOrphan();
|
|
1437
|
+
if (autoCancelResult?.cancelled) {
|
|
1438
|
+
bot._log(`[MAINT] Auto-cancelled unmatched price-drift orphan ${autoCancelResult.orderId} during ${context} ` +
|
|
1439
|
+
`(unblocking CREATE pipeline that targeted-drift reconcile could not adopt).`, 'warn');
|
|
1440
|
+
}
|
|
1441
|
+
else if (autoCancelResult?.reason) {
|
|
1442
|
+
bot._log(`[MAINT] Skipped unmatched-orphan auto-cancel during ${context}: ${autoCancelResult.reason}`, 'debug');
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1371
1445
|
refreshDynamicWeightDistribution(bot, context);
|
|
1372
1446
|
const dustCancelResult = await cancelDustOrders(bot, {
|
|
1373
1447
|
buy: healthResult.buyDustOrders,
|
|
@@ -1419,11 +1493,20 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
1419
1493
|
// immediately instead of patching master types. The retry re-fetches
|
|
1420
1494
|
// blockchain state and re-derives the boundary with consistent data.
|
|
1421
1495
|
// Skip spread correction this tick — the retry will handle it.
|
|
1422
|
-
|
|
1496
|
+
// EXCEPTION: a RECOVERY_EXHAUSTED abort means CREATEs are blocked until
|
|
1497
|
+
// the next fill or sync cycle (see recovery-exhausted reset logic), so an
|
|
1498
|
+
// immediate retry can never make progress and would spin the loop,
|
|
1499
|
+
// producing unbounded log volume. Defer to the normal cycle instead.
|
|
1500
|
+
if (dcResult && !dcResult.committed && dcResult.boundaryChanged &&
|
|
1501
|
+
dcResult.reason !== 'RECOVERY_EXHAUSTED') {
|
|
1423
1502
|
bot._log(`[DIVERGENCE-COW] Commit failed with boundary shift; ` +
|
|
1424
1503
|
`scheduling immediate retry instead of patching master`, 'warn');
|
|
1425
1504
|
setTimeout(() => runGridMaintenance(bot, 'failed-commit-retry', { skipIdle: true }), 0);
|
|
1426
1505
|
}
|
|
1506
|
+
else if (dcResult && !dcResult.committed && dcResult.boundaryChanged) {
|
|
1507
|
+
bot._log(`[DIVERGENCE-COW] Boundary-shift commit blocked (reason=${dcResult.reason}); ` +
|
|
1508
|
+
`deferring retry to next fill/sync cycle`, 'warn');
|
|
1509
|
+
}
|
|
1427
1510
|
else {
|
|
1428
1511
|
const spreadResult = await bot.manager.checkSpreadCondition(BitShares, bot.updateOrdersOnChainPlan.bind(bot));
|
|
1429
1512
|
if (await bot._abortFlowIfIllegalState(`${context} spread check`))
|
|
@@ -1448,13 +1531,14 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
1448
1531
|
}
|
|
1449
1532
|
}
|
|
1450
1533
|
/**
|
|
1451
|
-
* Cancel a single order,
|
|
1452
|
-
* credential daemon
|
|
1453
|
-
*
|
|
1454
|
-
*
|
|
1455
|
-
*
|
|
1456
|
-
*
|
|
1457
|
-
*
|
|
1534
|
+
* Cancel a single order, deferring on an uncertain daemon broadcast.
|
|
1535
|
+
* The credential daemon retries internally only failures that provably never
|
|
1536
|
+
* reached the chain (pre-transmit connect/send errors); on BROADCAST_DEADLINE
|
|
1537
|
+
* (or any uncertain outcome) it never re-signs, and the credential client
|
|
1538
|
+
* propagates a BroadcastUncertainError — the outcome is unknown and re-sending
|
|
1539
|
+
* could duplicate a landed cancel. No node is blacklisted here: without an
|
|
1540
|
+
* explicit nodeUrl the daemon chose the node, so there is no single node to
|
|
1541
|
+
* blame. The next dust-detection cycle re-attempts.
|
|
1458
1542
|
*
|
|
1459
1543
|
* The daemon's session is validated early in processRequest (before the
|
|
1460
1544
|
* broadcast), so a BROADCAST_DEADLINE reply cannot be caused by an expired
|
|
@@ -1464,23 +1548,13 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
1464
1548
|
* @param {import('./types').Order} order
|
|
1465
1549
|
* @returns {Promise<*>} Result from chainOrders.cancelOrder
|
|
1466
1550
|
*/
|
|
1467
|
-
async function
|
|
1551
|
+
async function cancelOrderDeferredOnUncertain(bot, order) {
|
|
1468
1552
|
try {
|
|
1469
|
-
|
|
1470
|
-
const nodes = nodeManager?.getHealthyNodes() ?? [];
|
|
1471
|
-
const fallbackNodes = nodes.length > 1 ? nodes.slice(1) : undefined;
|
|
1472
|
-
return await chainOrders.cancelOrder(bot.account, bot.privateKey, order.orderId, fallbackNodes
|
|
1473
|
-
? {
|
|
1474
|
-
fallbackNodes,
|
|
1475
|
-
onNodeFailed: (nodeUrl) => {
|
|
1476
|
-
nodeManager.reportNodeFailure(nodeUrl, 'BROADCAST_DEADLINE', 'broadcast');
|
|
1477
|
-
},
|
|
1478
|
-
}
|
|
1479
|
-
: {});
|
|
1553
|
+
return await chainOrders.cancelOrder(bot.account, bot.privateKey, order.orderId);
|
|
1480
1554
|
}
|
|
1481
1555
|
catch (err) {
|
|
1482
1556
|
if (err instanceof BroadcastUncertainError) {
|
|
1483
|
-
bot._warn(`[DUST]
|
|
1557
|
+
bot._warn(`[DUST] Broadcast uncertain for ${order.id} (${order.orderId}); outcome deferred to chain verification`);
|
|
1484
1558
|
}
|
|
1485
1559
|
throw err;
|
|
1486
1560
|
}
|
|
@@ -1505,12 +1579,27 @@ async function cancelDustOrders(bot, { buy: buyDust = [], sell: sellDust = [] }
|
|
|
1505
1579
|
if (!order.orderId)
|
|
1506
1580
|
continue;
|
|
1507
1581
|
try {
|
|
1508
|
-
const cancelResult = await
|
|
1582
|
+
const cancelResult = await cancelOrderDeferredOnUncertain(bot, order);
|
|
1509
1583
|
try {
|
|
1510
1584
|
if (cancelResult?.verifiedAfterFailure) {
|
|
1511
1585
|
const accountRef = bot.accountId || bot.account;
|
|
1512
|
-
|
|
1513
|
-
|
|
1586
|
+
// The cancel was verified absent on an authoritative
|
|
1587
|
+
// (non-empty, non-truncated) read inside cancelOrder, so an
|
|
1588
|
+
// ambiguous refetch must not run the full sync: the snapshot
|
|
1589
|
+
// omits the freshest orders and pass-1 phantom cleanup could
|
|
1590
|
+
// virtualize live slots. Fall back to the local cancel sync.
|
|
1591
|
+
const freshOrders = await readOpenOrdersGuarded(chainOrders, accountRef, {
|
|
1592
|
+
log: (message) => bot._warn(message),
|
|
1593
|
+
label: 'DUST',
|
|
1594
|
+
deferEmpty: true,
|
|
1595
|
+
skipMessage: (kind) => `[DUST] Chain refetch after verified cancel is ${kind}; applying local cancel sync for ${order.id}`,
|
|
1596
|
+
});
|
|
1597
|
+
if (freshOrders === null) {
|
|
1598
|
+
await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
|
|
1599
|
+
}
|
|
1600
|
+
else {
|
|
1601
|
+
await bot.manager.synchronizeWithChain(freshOrders, 'readOpenOrders');
|
|
1602
|
+
}
|
|
1514
1603
|
}
|
|
1515
1604
|
else {
|
|
1516
1605
|
await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
|
|
@@ -1714,7 +1803,7 @@ async function acquireBts(bot, deficit) {
|
|
|
1714
1803
|
}
|
|
1715
1804
|
const orderType = (best.asset.id === bot.assets?.assetA?.id) ? 'sell' : 'buy';
|
|
1716
1805
|
if (bot.manager.accountant) {
|
|
1717
|
-
bot.manager.accountant.adjustTotalBalance(orderType, -best.sellAmount, 'bts-acquisition-swap-sell');
|
|
1806
|
+
await bot.manager.accountant.adjustTotalBalance(orderType, -best.sellAmount, 'bts-acquisition-swap-sell');
|
|
1718
1807
|
}
|
|
1719
1808
|
// Do NOT optimistically bump btsBalance.free/total here. expectedReceive is
|
|
1720
1809
|
// a pre-swap estimate and may diverge from the actual fill (slippage, fees,
|
|
@@ -1781,7 +1870,7 @@ function setupDustHealthCheckInterval(bot) {
|
|
|
1781
1870
|
* Request a full grid reset from fresh on-chain state.
|
|
1782
1871
|
* @param {import('./dexbot_class').DEXBot} bot
|
|
1783
1872
|
* @param {string} [reason='structural change']
|
|
1784
|
-
* @param {{refreshCenterPrice?: boolean}} [options={}]
|
|
1873
|
+
* @param {{refreshCenterPrice?: boolean, skipIdle?: boolean, skipFillLock?: boolean}} [options={}]
|
|
1785
1874
|
* @returns {Promise<Object>}
|
|
1786
1875
|
*/
|
|
1787
1876
|
async function requestGridReset(bot, reason = 'structural change', options = {}) {
|
|
@@ -1794,8 +1883,21 @@ async function requestGridReset(bot, reason = 'structural change', options = {})
|
|
|
1794
1883
|
...options,
|
|
1795
1884
|
refreshCenterPrice: options.refreshCenterPrice !== false,
|
|
1796
1885
|
};
|
|
1797
|
-
|
|
1798
|
-
|
|
1886
|
+
// Structural recovery resyncs are chain-read/rebuild operations that do not
|
|
1887
|
+
// mutate order state in place; running them behind the fill-processing lock
|
|
1888
|
+
// lets a fill storm starve recovery indefinitely. Skip the lock when the
|
|
1889
|
+
// caller explicitly requests it (structural resync wiring). While the
|
|
1890
|
+
// rebuild runs unlocked, raise _recoverySyncInFlight so the fill consumer
|
|
1891
|
+
// defers fills entirely (no concurrent mutation race) instead of racing
|
|
1892
|
+
// them behind a lock.
|
|
1893
|
+
if (options.skipFillLock === true || !bot.manager._fillProcessingLock || bot.manager._fillProcessingLock.isReentrant()) {
|
|
1894
|
+
bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
|
|
1895
|
+
try {
|
|
1896
|
+
return await performGridResync(bot, resetOptions);
|
|
1897
|
+
}
|
|
1898
|
+
finally {
|
|
1899
|
+
bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
|
|
1900
|
+
}
|
|
1799
1901
|
}
|
|
1800
1902
|
return bot.manager._fillProcessingLock.acquire(async () => performGridResync(bot, resetOptions));
|
|
1801
1903
|
}
|
|
@@ -1816,13 +1918,36 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
1816
1918
|
const unmatchedCount = Array.isArray(details?.unmatchedChainOrders)
|
|
1817
1919
|
? details.unmatchedChainOrders.length
|
|
1818
1920
|
: 0;
|
|
1819
|
-
|
|
1921
|
+
const runStructuralResync = async () => {
|
|
1820
1922
|
bot._structuralGridResyncTimer = null;
|
|
1821
1923
|
if (bot._shuttingDown)
|
|
1822
1924
|
return;
|
|
1925
|
+
// A COW batch can be mid-broadcast (its broadcast phase holds no
|
|
1926
|
+
// grid lock, only the commit does). Reloading the persisted grid
|
|
1927
|
+
// underneath it would race the commit and force an avoidable
|
|
1928
|
+
// commit refusal + re-adoption cycle. Defer until the batch
|
|
1929
|
+
// completes; the fill consumer is already gated on _batchInFlight,
|
|
1930
|
+
// so this only blocks the recovery itself, never new fills.
|
|
1931
|
+
if (bot._batchInFlight > 0) {
|
|
1932
|
+
bot._structuralGridResyncTimer = setTimeout(runStructuralResync, TIMING.LOCK_REFRESH_MIN_MS);
|
|
1933
|
+
return;
|
|
1934
|
+
}
|
|
1823
1935
|
bot._structuralGridResyncRunning++;
|
|
1824
1936
|
try {
|
|
1825
|
-
|
|
1937
|
+
// The reload runs without the fill lock, so raise the
|
|
1938
|
+
// recovery-in-flight flag for its whole duration: the fill
|
|
1939
|
+
// consumer defers (dexbot_fill_runtime gates on it), which
|
|
1940
|
+
// prevents a fill arriving mid-reload from starting a new COW
|
|
1941
|
+
// batch whose broadcast would overlap the reload and force an
|
|
1942
|
+
// avoidable commit refusal + re-adoption cycle.
|
|
1943
|
+
bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
|
|
1944
|
+
let persistedResult;
|
|
1945
|
+
try {
|
|
1946
|
+
persistedResult = await bot._recoverFromPersistedGrid();
|
|
1947
|
+
}
|
|
1948
|
+
finally {
|
|
1949
|
+
bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
|
|
1950
|
+
}
|
|
1826
1951
|
if (persistedResult.success) {
|
|
1827
1952
|
if (bot.manager?._recoveryState) {
|
|
1828
1953
|
bot.manager._recoveryState = { ...bot.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
|
|
@@ -1833,6 +1958,11 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
1833
1958
|
bot._warn(`[RECOVERY] Running structural full grid resync for ${reason}${suffix}`);
|
|
1834
1959
|
const resetResult = await bot.requestGridReset('rms_structural_grid_resync', {
|
|
1835
1960
|
refreshCenterPrice: true,
|
|
1961
|
+
// Structural resync is a chain-read/rebuild; do not let a
|
|
1962
|
+
// fill storm starve it behind the idle cooldown or the
|
|
1963
|
+
// fill-processing lock.
|
|
1964
|
+
skipIdle: true,
|
|
1965
|
+
skipFillLock: true,
|
|
1836
1966
|
});
|
|
1837
1967
|
if (resetResult && bot.manager?._recoveryState) {
|
|
1838
1968
|
bot.manager._recoveryState = { ...bot.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
|
|
@@ -1847,7 +1977,8 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
1847
1977
|
bot.manager._recoveryState = { ...bot.manager._recoveryState, structuralResyncRequested: false };
|
|
1848
1978
|
}
|
|
1849
1979
|
}
|
|
1850
|
-
}
|
|
1980
|
+
};
|
|
1981
|
+
bot._structuralGridResyncTimer = setTimeout(runStructuralResync, 0);
|
|
1851
1982
|
return { scheduled: true };
|
|
1852
1983
|
};
|
|
1853
1984
|
}
|
|
@@ -1903,7 +2034,19 @@ async function syncOpenOrdersAndProcessFills(bot, tag) {
|
|
|
1903
2034
|
return { syncResult: null, aborted: false, hasUnmatched: 0, openOrders: null };
|
|
1904
2035
|
}
|
|
1905
2036
|
try {
|
|
1906
|
-
|
|
2037
|
+
// Truncated-read guard: syncing on a partial get_full_accounts window
|
|
2038
|
+
// would virtualize live ACTIVE slots (pass-1 phantom cleanup) and
|
|
2039
|
+
// re-create them as duplicates. Defer to a clean read — fill
|
|
2040
|
+
// subscription events keep the bot responsive in the meantime.
|
|
2041
|
+
const firstRead = await readOpenOrdersGuarded(chainOrders, bot.accountId, {
|
|
2042
|
+
log: (message, level) => bot._log(message, level),
|
|
2043
|
+
label: 'SYNC-CHAIN',
|
|
2044
|
+
detail: `during ${tag}`,
|
|
2045
|
+
});
|
|
2046
|
+
if (firstRead === null) {
|
|
2047
|
+
return { syncResult: null, aborted: false, hasUnmatched: 0, openOrders: null };
|
|
2048
|
+
}
|
|
2049
|
+
let openOrders = firstRead;
|
|
1907
2050
|
const syncResult = await bot.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
|
|
1908
2051
|
let aborted = false;
|
|
1909
2052
|
if (syncResult?.filledOrders?.length > 0) {
|
|
@@ -1911,8 +2054,18 @@ async function syncOpenOrdersAndProcessFills(bot, tag) {
|
|
|
1911
2054
|
bot._log(`[SYNC-CHAIN] ${syncResult.filledOrders.length} filled order(s) found during ${tag}`, 'info');
|
|
1912
2055
|
const batchResult = await bot._processFillsWithBatching(syncResult.filledOrders, new Set(), `${tag} sync-fill`);
|
|
1913
2056
|
if (!batchResult?.aborted) {
|
|
1914
|
-
|
|
1915
|
-
|
|
2057
|
+
// Reassign the returned snapshot to the post-fill re-read: the
|
|
2058
|
+
// caller feeds openOrders into reconcileGridOrders, which must
|
|
2059
|
+
// see the freshest chain state, not the pre-fill first read.
|
|
2060
|
+
const reReadOrders = await readOpenOrdersGuarded(chainOrders, bot.accountId, {
|
|
2061
|
+
log: (message, level) => bot._log(message, level),
|
|
2062
|
+
label: 'SYNC-CHAIN',
|
|
2063
|
+
detail: `post-fill re-read during ${tag}`,
|
|
2064
|
+
});
|
|
2065
|
+
if (reReadOrders !== null) {
|
|
2066
|
+
openOrders = reReadOrders;
|
|
2067
|
+
await bot.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
|
|
2068
|
+
}
|
|
1916
2069
|
}
|
|
1917
2070
|
else {
|
|
1918
2071
|
aborted = true;
|