dexbot 1.7.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +3 -3
- package/analysis/README.md +33 -13
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/account_resolver.js +5 -6
- package/dist/analysis/analyze_liquidity.js +15 -4
- package/dist/analysis/bot_key_utils.d.ts +44 -1
- package/dist/analysis/bot_key_utils.js +76 -7
- package/dist/analysis/fills_cache.js +2 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -19
- package/dist/analysis/grid_correction_check.js +12 -30
- package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
- package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
- package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
- package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
- package/dist/analysis/liquidity/liquidity_view.js +11 -0
- package/dist/analysis/resolve_bot_accounts.js +7 -6
- package/dist/analysis/trade_profitability.js +2 -1
- package/dist/credential-daemon.js +13 -2
- package/dist/dexbot.js +4 -4
- package/dist/modules/account_bots.d.ts +11 -1
- package/dist/modules/account_orders.d.ts +12 -0
- package/dist/modules/account_orders.js +30 -0
- package/dist/modules/bitshares-native/transport.js +24 -6
- package/dist/modules/bitshares_client.d.ts +24 -6
- package/dist/modules/bitshares_client.js +44 -7
- package/dist/modules/bot_settings.d.ts +2 -0
- package/dist/modules/constants.d.ts +26 -3
- package/dist/modules/constants.js +63 -8
- package/dist/modules/dexbot_class.d.ts +1 -6
- package/dist/modules/dexbot_class.js +14 -0
- package/dist/modules/dexbot_cow_runtime.d.ts +97 -7
- package/dist/modules/dexbot_cow_runtime.js +620 -132
- package/dist/modules/dexbot_fill_runtime.js +124 -35
- package/dist/modules/dexbot_maintenance_runtime.js +226 -73
- package/dist/modules/dexbot_startup_runtime.js +32 -20
- package/dist/modules/dexbot_state_recovery.d.ts +33 -1
- package/dist/modules/dexbot_state_recovery.js +167 -5
- package/dist/modules/node_connect_policy.d.ts +8 -1
- package/dist/modules/node_connect_policy.js +18 -2
- package/dist/modules/node_flap_score.d.ts +87 -0
- package/dist/modules/node_flap_score.js +124 -0
- package/dist/modules/node_health_cache.d.ts +40 -3
- package/dist/modules/node_health_cache.js +66 -20
- package/dist/modules/node_manager.d.ts +101 -19
- package/dist/modules/node_manager.js +255 -65
- package/dist/modules/order/accounting.js +56 -7
- package/dist/modules/order/format.d.ts +24 -10
- package/dist/modules/order/format.js +30 -10
- package/dist/modules/order/grid.js +82 -11
- package/dist/modules/order/grid_reconcile.d.ts +5 -1
- package/dist/modules/order/grid_reconcile.js +333 -210
- package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
- package/dist/modules/order/grid_reconcile_internal.js +207 -87
- package/dist/modules/order/index.d.ts +4 -0
- package/dist/modules/order/lifecycle.d.ts +353 -0
- package/dist/modules/order/lifecycle.js +1109 -0
- package/dist/modules/order/manager.d.ts +271 -5
- package/dist/modules/order/manager.js +651 -48
- package/dist/modules/order/placement_lock.d.ts +102 -0
- package/dist/modules/order/placement_lock.js +222 -0
- package/dist/modules/order/strategy.js +13 -1
- package/dist/modules/order/sync_engine.d.ts +13 -1
- package/dist/modules/order/sync_engine.js +153 -53
- package/dist/modules/order/utils/math.d.ts +22 -1
- package/dist/modules/order/utils/math.js +54 -1
- package/dist/modules/order/utils/order.d.ts +75 -5
- package/dist/modules/order/utils/order.js +158 -9
- package/dist/modules/order/utils/system.d.ts +18 -0
- package/dist/modules/order/utils/system.js +84 -8
- package/dist/modules/order/utils/validate.d.ts +7 -1
- package/dist/modules/order/utils/validate.js +48 -2
- package/dist/modules/outcome.d.ts +54 -0
- package/dist/modules/outcome.js +51 -0
- package/dist/modules/settings_merge.d.ts +4 -0
- package/dist/modules/settings_merge.js +9 -0
- package/dist/modules/timing_policy.d.ts +82 -0
- package/dist/modules/timing_policy.js +244 -0
- package/dist/modules/types.d.ts +132 -0
- package/dist/scripts/analyze-credit.js +54 -17
- package/dist/scripts/analyze-orders.d.ts +4 -2
- package/dist/scripts/analyze-orders.js +113 -34
- package/dist/scripts/chart_targets.js +11 -15
- package/dist/scripts/test-credit-renewal.js +12 -5
- package/docs/COW_INVARIANTS.md +67 -8
- package/docs/DEXBOT_COMPARISON.md +6 -6
- package/docs/EVOLUTION.md +9 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_PRICE_INVARIANT.md +1 -1
- package/docs/GRID_RECONCILE.md +17 -12
- package/docs/LOGGING.md +25 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +175 -26
- package/docs/README.md +23 -2
- package/docs/WORKFLOW.md +9 -3
- package/docs/architecture.md +42 -5
- package/docs/developer_guide.md +25 -8
- package/modules/README.md +3 -0
- package/package.json +1 -1
- package/scripts/README.md +16 -0
- package/scripts/pdev.sh +14 -1
- package/scripts/pmain.sh +23 -8
- package/scripts/ptest.sh +7 -0
- package/tests/README.md +30 -1
|
@@ -9,6 +9,7 @@ import { lazyShim } from './lazy_shim.js';
|
|
|
9
9
|
import fs from 'node:fs';
|
|
10
10
|
import { spawn } from 'node:child_process';
|
|
11
11
|
import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills } from './order/utils/system.js';
|
|
12
|
+
import { withPlacement } from './order/placement_lock.js';
|
|
12
13
|
import { path } from './path_api.js';
|
|
13
14
|
import * as chainOrders from './chain_orders.js';
|
|
14
15
|
import * as grid from './order/grid.js';
|
|
@@ -29,6 +30,11 @@ import { BroadcastUncertainError } from './dexbot_credential_client.js';
|
|
|
29
30
|
import * as configModule from './config.js';
|
|
30
31
|
const { Config } = configModule;
|
|
31
32
|
import { getErrorMessage } from './utils/errors.js';
|
|
33
|
+
/**
|
|
34
|
+
* Gap-evac stranding sweep (lazy import: the recovery module pulls the
|
|
35
|
+
* bitshares client into the graph, which this runtime loads on demand already).
|
|
36
|
+
*/
|
|
37
|
+
const scheduleGapEvacRepair = lazyShim('./dexbot_state_recovery', 'scheduleGapEvacRepair');
|
|
32
38
|
import { isSameBotName } from './utils/sanitize_key.js';
|
|
33
39
|
import { usesAmaGridPrice } from './grid_price_source.js';
|
|
34
40
|
const hasOpenOrdersSyncLoopMsSet = lazyShim('./config', 'hasOpenOrdersSyncLoopMsSet');
|
|
@@ -42,6 +48,7 @@ const applyGridDivergenceCorrections = lazyShim('./order/utils/system', 'applyGr
|
|
|
42
48
|
const updateGridFromBlockchainSnapshot = lazyShim('./order/grid', 'updateGridFromBlockchainSnapshot');
|
|
43
49
|
const loadAmaCenterSnapshot = lazyShim('./order/utils/system', 'loadAmaCenterSnapshot');
|
|
44
50
|
const sleep = lazyShim('./order/utils/system', 'sleep');
|
|
51
|
+
const getPipelineInFlightAgeMs = lazyShim('./order/utils/system', 'getPipelineInFlightAgeMs');
|
|
45
52
|
const parseJsonWithComments = lazyShim('./order/utils/system', 'parseJsonWithComments');
|
|
46
53
|
const isPm2Runtime = lazyShim('./order/logger', 'isPm2Runtime');
|
|
47
54
|
const isWrapperAdapterOwner = lazyShim('./launcher/adapter_requirement', 'isWrapperAdapterOwner');
|
|
@@ -70,6 +77,7 @@ const isStrandedHoldOrder = lazyShim('./order/utils/order', 'isStrandedHoldOrder
|
|
|
70
77
|
const getSideBudget = lazyShim('./order/utils/order', 'getSideBudget');
|
|
71
78
|
function getCoreOrdersTotal(config) { return require('./order/utils/order').getCoreOrdersTotal(config); }
|
|
72
79
|
const correctAllPriceMismatches = lazyShim('./order/utils/order', 'correctAllPriceMismatches');
|
|
80
|
+
const getLivePendingBroadcasts = lazyShim('./order/utils/order', 'getLivePendingBroadcasts');
|
|
73
81
|
const isOrderOnChain = lazyShim('./order/utils/order', 'isOrderOnChain');
|
|
74
82
|
const parseChainOrder = lazyShim('./order/utils/order', 'parseChainOrder');
|
|
75
83
|
const parseSlotIndex = lazyShim('./order/utils/slot', 'parseSlotIndex');
|
|
@@ -1315,6 +1323,10 @@ function performGridResync(bot, options = {}) {
|
|
|
1315
1323
|
return Promise.resolve(false);
|
|
1316
1324
|
}
|
|
1317
1325
|
self.manager.startBootstrap();
|
|
1326
|
+
// Set BEFORE the async work so a fill-driven COW that starts while the
|
|
1327
|
+
// resync reloads config/re-derives the centre already sees the rebuild and
|
|
1328
|
+
// defers. recalculateGrid adds a second (balanced) level for its own body.
|
|
1329
|
+
self.manager.beginStructuralRebuild?.('trigger-file resync');
|
|
1318
1330
|
self._log('Grid regeneration triggered. Performing full grid resync...');
|
|
1319
1331
|
return (async () => {
|
|
1320
1332
|
try {
|
|
@@ -1436,6 +1448,7 @@ function performGridResync(bot, options = {}) {
|
|
|
1436
1448
|
}
|
|
1437
1449
|
finally {
|
|
1438
1450
|
self.manager.finishBootstrap();
|
|
1451
|
+
self.manager.endStructuralRebuild?.();
|
|
1439
1452
|
}
|
|
1440
1453
|
return success;
|
|
1441
1454
|
})();
|
|
@@ -2265,6 +2278,77 @@ function scheduleDeferredGridResync(ctx, options = {}) {
|
|
|
2265
2278
|
});
|
|
2266
2279
|
}, delayMs);
|
|
2267
2280
|
}
|
|
2281
|
+
/**
|
|
2282
|
+
* Raise/lower the recovery-sync in-flight counter as a pair that also
|
|
2283
|
+
* maintains `_recoverySyncInFlightStartedAt`. The leak watchdog keys off the
|
|
2284
|
+
* timestamp: incrementing without it makes the watchdog skip the window (or
|
|
2285
|
+
* use a stale timestamp and force-clear a live recovery).
|
|
2286
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
2287
|
+
*/
|
|
2288
|
+
function raiseRecoveryInFlight(bot) {
|
|
2289
|
+
bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
|
|
2290
|
+
bot._recoverySyncInFlightStartedAt = Date.now();
|
|
2291
|
+
}
|
|
2292
|
+
/** @param {import('./dexbot_class.js').DEXBot} bot */
|
|
2293
|
+
function lowerRecoveryInFlight(bot) {
|
|
2294
|
+
bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
|
|
2295
|
+
if ((bot._recoverySyncInFlight || 0) === 0) {
|
|
2296
|
+
bot._recoverySyncInFlightStartedAt = 0;
|
|
2297
|
+
drainFillQueueAfterPipelineClear(bot);
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2300
|
+
/**
|
|
2301
|
+
* Leak watchdog for the bot-level pipeline counters. A missed decrement parks
|
|
2302
|
+
* the fill queue indefinitely; there is no stale-clear equivalent for
|
|
2303
|
+
* _batchInFlight / _recoverySyncInFlight. Surface the age every tick, and only
|
|
2304
|
+
* force-clear when nothing actually holds the placement region (a live batch
|
|
2305
|
+
* owns it, so resetting there would allow a second concurrent broadcast).
|
|
2306
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
2307
|
+
*/
|
|
2308
|
+
function checkPipelineInFlightWatchdogs(bot) {
|
|
2309
|
+
const now = Date.now();
|
|
2310
|
+
const placementHeld = typeof bot.manager?.isPlacementActive === 'function'
|
|
2311
|
+
&& bot.manager.isPlacementActive() === true;
|
|
2312
|
+
const checks = [
|
|
2313
|
+
{
|
|
2314
|
+
countKey: '_batchInFlight',
|
|
2315
|
+
startedKey: '_batchInFlightStartedAt',
|
|
2316
|
+
label: 'batch',
|
|
2317
|
+
staleMs: Number(TIMING.BATCH_INFLIGHT_STALE_MS) > 0 ? Number(TIMING.BATCH_INFLIGHT_STALE_MS) : 10 * 60 * 1000,
|
|
2318
|
+
},
|
|
2319
|
+
{
|
|
2320
|
+
countKey: '_recoverySyncInFlight',
|
|
2321
|
+
startedKey: '_recoverySyncInFlightStartedAt',
|
|
2322
|
+
label: 'recovery sync',
|
|
2323
|
+
staleMs: Number(TIMING.RECOVERY_INFLIGHT_STALE_MS) > 0 ? Number(TIMING.RECOVERY_INFLIGHT_STALE_MS) : 10 * 60 * 1000,
|
|
2324
|
+
},
|
|
2325
|
+
];
|
|
2326
|
+
const botAny = bot;
|
|
2327
|
+
for (const check of checks) {
|
|
2328
|
+
const count = Number(botAny[check.countKey]) || 0;
|
|
2329
|
+
if (count <= 0)
|
|
2330
|
+
continue;
|
|
2331
|
+
const startedAt = Number(botAny[check.startedKey]) || 0;
|
|
2332
|
+
if (!startedAt)
|
|
2333
|
+
continue;
|
|
2334
|
+
const ageMs = now - startedAt;
|
|
2335
|
+
if (ageMs < check.staleMs)
|
|
2336
|
+
continue;
|
|
2337
|
+
if (placementHeld) {
|
|
2338
|
+
bot._warn(`[WATCHDOG] ${check.label} counter (${count}) has been in flight for ${Math.round(ageMs / 1000)}s ` +
|
|
2339
|
+
`but the placement region is still held by ${bot.manager.placementKind?.() || 'unknown'}; not force-clearing`);
|
|
2340
|
+
continue;
|
|
2341
|
+
}
|
|
2342
|
+
bot._warn(`[WATCHDOG] Force-clearing leaked ${check.label} counter (${count}) after ${Math.round(ageMs / 1000)}s ` +
|
|
2343
|
+
`with no placement holder; the pipeline must not stay parked`);
|
|
2344
|
+
botAny[check.countKey] = 0;
|
|
2345
|
+
botAny[check.startedKey] = 0;
|
|
2346
|
+
// Wake the fill consumer: the deferred branch gates on this counter
|
|
2347
|
+
// without rescheduling, so a force-clear that does not drain would
|
|
2348
|
+
// leave the queue parked until an unrelated event.
|
|
2349
|
+
drainFillQueueAfterPipelineClear(bot);
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2268
2352
|
/**
|
|
2269
2353
|
* Execute the core maintenance logic: recalculate funds, check pipeline,
|
|
2270
2354
|
* refresh dynamic weights, check grid health, cancel dust orders,
|
|
@@ -2277,10 +2361,21 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
2277
2361
|
// Surface any over-long _gridLock hold first — observational only, and it
|
|
2278
2362
|
// must run even when the rest of the tick defers on a broadcast region.
|
|
2279
2363
|
checkGridLockHoldDuration(bot);
|
|
2280
|
-
//
|
|
2281
|
-
//
|
|
2282
|
-
//
|
|
2364
|
+
// Placement leak watchdog first: it is the authoritative mutex and the one
|
|
2365
|
+
// authority that may tear a hung region down (it also drops that region's
|
|
2366
|
+
// stale broadcast refcount atomically). A live pipeline counter younger than
|
|
2367
|
+
// the bound keeps the region alive, so a slow create-group reconcile is not
|
|
2368
|
+
// cut out from under its own leases.
|
|
2369
|
+
try {
|
|
2370
|
+
bot.manager?._clearStalePlacementRegion?.(undefined, {
|
|
2371
|
+
batchInFlightMs: getPipelineInFlightAgeMs(bot),
|
|
2372
|
+
});
|
|
2373
|
+
}
|
|
2374
|
+
catch { /* observational */ }
|
|
2375
|
+
// Broadcast-flag watchdog: it refuses to act while a placement lease is
|
|
2376
|
+
// held, so it only clears regions that never took the exclusive region.
|
|
2283
2377
|
bot.manager._clearStaleBroadcastFlag();
|
|
2378
|
+
checkPipelineInFlightWatchdogs(bot);
|
|
2284
2379
|
await bot.manager.recalculateFunds();
|
|
2285
2380
|
await checkBtsBalanceAndAcquire(bot);
|
|
2286
2381
|
bot.manager.clearStalePipelineOperations();
|
|
@@ -2326,9 +2421,19 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
2326
2421
|
// with batch broadcasts — transient mismatches are expected during a COW cycle.
|
|
2327
2422
|
// Also skip while a structural recovery reload is in flight (it runs without
|
|
2328
2423
|
// the fill lock, so this check must gate on the recovery flag explicitly).
|
|
2424
|
+
// Liveness count from the single authority; a test double that does not
|
|
2425
|
+
// model it falls back to the raw map size (a COUNT only — never a second
|
|
2426
|
+
// TTL policy), so a partial mock cannot make the gate throw.
|
|
2427
|
+
let pendingBroadcastCount = 0;
|
|
2428
|
+
try {
|
|
2429
|
+
pendingBroadcastCount = Number(getLivePendingBroadcasts(bot.manager)?.length) || 0;
|
|
2430
|
+
}
|
|
2431
|
+
catch {
|
|
2432
|
+
pendingBroadcastCount = Number(bot.manager?._pendingBroadcasts?.size) || 0;
|
|
2433
|
+
}
|
|
2329
2434
|
if (!bot._recoverySyncInFlight &&
|
|
2330
2435
|
!bot._batchInFlight &&
|
|
2331
|
-
|
|
2436
|
+
pendingBroadcastCount === 0 &&
|
|
2332
2437
|
(bot._lightweightSyncCheckAt == null || Date.now() - bot._lightweightSyncCheckAt >= TIMING.LIGHTWEIGHT_SYNC_CHECK_INTERVAL_MS)) {
|
|
2333
2438
|
bot._lightweightSyncCheckAt = Date.now();
|
|
2334
2439
|
try {
|
|
@@ -2368,6 +2473,22 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
2368
2473
|
bot._log(`[LIGHTWEIGHT-SYNC] Check failed: ${getErrorMessage(e)}`, 'debug');
|
|
2369
2474
|
}
|
|
2370
2475
|
}
|
|
2476
|
+
// GAP-EVAC stranding sweep: a boundary commit that moved away from a live
|
|
2477
|
+
// order strands it inside the spread band, and plans are fill-driven — so a
|
|
2478
|
+
// quiet market never heals it (the band keeps live orders and the real
|
|
2479
|
+
// spread stays collapsed). The sweep re-derives the streak ledger against
|
|
2480
|
+
// the COMMITTED geometry, queues the cancel-only teeth for anything past
|
|
2481
|
+
// the threshold, and schedules a fill-less re-plan (the preferred repair:
|
|
2482
|
+
// one in-place price update). It early-returns on a healthy band, so it is
|
|
2483
|
+
// cheap enough to run every tick, and it also covers commits made by paths
|
|
2484
|
+
// that hold no bot reference (divergence/sync) plus a restart that loads an
|
|
2485
|
+
// already-stranded snapshot.
|
|
2486
|
+
try {
|
|
2487
|
+
scheduleGapEvacRepair(bot, `maintenance ${context}`);
|
|
2488
|
+
}
|
|
2489
|
+
catch (gapEvacErr) {
|
|
2490
|
+
bot._log(`[GAP-EVAC] Stranding sweep failed (non-fatal): ${getErrorMessage(gapEvacErr)}`, 'debug');
|
|
2491
|
+
}
|
|
2371
2492
|
// Process any price corrections queued by prior sync operations before
|
|
2372
2493
|
// the pipeline gate. Pending corrections block isPipelineEmpty, and no
|
|
2373
2494
|
// other code path clears them outside of _consumeFillQueue (which only
|
|
@@ -2375,20 +2496,34 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
2375
2496
|
// during startup or periodic sync can stall the pipeline indefinitely.
|
|
2376
2497
|
const pendingCorrections = bot.manager.ordersNeedingPriceCorrection?.length || 0;
|
|
2377
2498
|
if (pendingCorrections > 0) {
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
bot.
|
|
2386
|
-
|
|
2387
|
-
(
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2499
|
+
// The drain cancels/updates chain orders directly, so it must own the
|
|
2500
|
+
// exclusive placement region. `correctAllPriceMismatches` acquires it
|
|
2501
|
+
// internally (single entry point); this call uses a short bound because
|
|
2502
|
+
// this is a background repair and must not block the maintenance tick
|
|
2503
|
+
// behind a long broadcast. A busy region leaves the queue for the next
|
|
2504
|
+
// tick (or the region-end wake-up).
|
|
2505
|
+
try {
|
|
2506
|
+
const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders, { timeoutMs: 250 });
|
|
2507
|
+
if (correctionResult?.deferred === true) {
|
|
2508
|
+
bot._log('[MAINT] Deferring price-correction drain: exclusive placement region busy or shutting down', 'info');
|
|
2509
|
+
}
|
|
2510
|
+
else if (correctionResult.failed > 0) {
|
|
2511
|
+
const failedDetails = (correctionResult.results || [])
|
|
2512
|
+
.filter((r) => !(r.result && r.result.success))
|
|
2513
|
+
.slice(0, 3)
|
|
2514
|
+
.map((r) => `${r.chainOrderId ?? '?'}: ${r.result?.error ?? 'unknown'}`)
|
|
2515
|
+
.join(' | ');
|
|
2516
|
+
bot._warn(`[MAINT] ${correctionResult.failed}/${pendingCorrections} price correction(s) failed` +
|
|
2517
|
+
(failedDetails ? ` — ${failedDetails}` : '') +
|
|
2518
|
+
(correctionResult.staleDropped > 0 ? ` (${correctionResult.staleDropped} stale dropped)` : ''));
|
|
2519
|
+
}
|
|
2520
|
+
else if (correctionResult.staleDropped > 0) {
|
|
2521
|
+
bot._log(`[MAINT] ${correctionResult.corrected}/${pendingCorrections} price correction(s) resolved, ` +
|
|
2522
|
+
`${correctionResult.staleDropped} stale dropped (resync moved the slot; re-queued by next sync if still off-target)`, 'info');
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
catch (correctionErr) {
|
|
2526
|
+
bot._warn(`[MAINT] Price-correction drain failed: ${getErrorMessage(correctionErr)}`);
|
|
2392
2527
|
}
|
|
2393
2528
|
}
|
|
2394
2529
|
// Dust detection runs before the pipeline gate. Cancellation is immediate
|
|
@@ -2764,6 +2899,14 @@ async function cancelOrderDeferredOnUncertain(bot, order) {
|
|
|
2764
2899
|
throw err;
|
|
2765
2900
|
}
|
|
2766
2901
|
}
|
|
2902
|
+
async function runWithPlacement(bot, kind, fn, options = {}) {
|
|
2903
|
+
// Bot-first adapter over the ONE acquire/release contract
|
|
2904
|
+
// (modules/order/placement_lock.ts): it owns the bounded wait, the shutdown
|
|
2905
|
+
// probe, the NOOP fallback for manager-less fakes and the release. Do not
|
|
2906
|
+
// re-implement that here — a local copy already drifted once (it could not
|
|
2907
|
+
// take a lease and skipped the lease-liveness check).
|
|
2908
|
+
return await withPlacement(bot.manager, kind, fn, options);
|
|
2909
|
+
}
|
|
2767
2910
|
/**
|
|
2768
2911
|
* Cancel dust orders immediately — no delay, no timer, no maps.
|
|
2769
2912
|
* Each dust order is cancelled on chain and its slot is rotated through
|
|
@@ -2780,52 +2923,63 @@ async function cancelDustOrders(bot, { buy: buyDust = [], sell: sellDust = [] }
|
|
|
2780
2923
|
if (allDust.length === 0)
|
|
2781
2924
|
return { cancelledCount: 0, batchResult: null };
|
|
2782
2925
|
const syntheticFills = [];
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2926
|
+
// Hold the exclusive placement region across the whole cancel pass: each
|
|
2927
|
+
// cancel mutates the chain and the following synchronizeWithChain mutates
|
|
2928
|
+
// the master. A COW batch / reconcile must not interleave between them.
|
|
2929
|
+
// The rotation batch below runs after release (it acquires its own lease).
|
|
2930
|
+
const placementOk = await runWithPlacement(bot, 'dust-cancel', async () => {
|
|
2931
|
+
for (const order of allDust) {
|
|
2932
|
+
if (!order.orderId)
|
|
2933
|
+
continue;
|
|
2788
2934
|
try {
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2935
|
+
const cancelResult = await cancelOrderDeferredOnUncertain(bot, order);
|
|
2936
|
+
try {
|
|
2937
|
+
if (cancelResult?.verifiedAfterFailure) {
|
|
2938
|
+
const accountRef = bot.accountId || bot.account;
|
|
2939
|
+
// The cancel was verified absent on an authoritative
|
|
2940
|
+
// (non-empty, non-truncated) read inside cancelOrder, so an
|
|
2941
|
+
// ambiguous refetch must not run the full sync: the snapshot
|
|
2942
|
+
// omits the freshest orders and pass-1 phantom cleanup could
|
|
2943
|
+
// virtualize live slots. Fall back to the local cancel sync.
|
|
2944
|
+
const freshOrders = await readOpenOrdersGuarded(chainOrders, accountRef, {
|
|
2945
|
+
log: (message) => bot._warn(message),
|
|
2946
|
+
label: 'DUST',
|
|
2947
|
+
deferEmpty: true,
|
|
2948
|
+
skipMessage: (kind) => `[DUST] Chain refetch after verified cancel is ${kind}; applying local cancel sync for ${order.id}`,
|
|
2949
|
+
});
|
|
2950
|
+
if (freshOrders === null) {
|
|
2951
|
+
await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
|
|
2952
|
+
}
|
|
2953
|
+
else {
|
|
2954
|
+
await bot.manager.synchronizeWithChain(freshOrders, 'readOpenOrders');
|
|
2955
|
+
}
|
|
2804
2956
|
}
|
|
2805
2957
|
else {
|
|
2806
|
-
await bot.manager.synchronizeWithChain(
|
|
2958
|
+
await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
|
|
2807
2959
|
}
|
|
2808
2960
|
}
|
|
2809
|
-
|
|
2810
|
-
|
|
2961
|
+
catch (refetchErr) {
|
|
2962
|
+
bot._warn(`[DUST] Cancel succeeded but refetch failed for ${order.id} (${order.orderId}): ${getErrorMessage(refetchErr)}`);
|
|
2811
2963
|
}
|
|
2812
|
-
}
|
|
2813
|
-
catch (refetchErr) {
|
|
2814
|
-
bot._warn(`[DUST] Cancel succeeded but refetch failed for ${order.id} (${order.orderId}): ${getErrorMessage(refetchErr)}`);
|
|
2815
|
-
}
|
|
2816
|
-
syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
|
|
2817
|
-
bot._log(`[DUST] Cancelled ${order.id} (${order.orderId}) size=${order.size}`, 'debug');
|
|
2818
|
-
}
|
|
2819
|
-
catch (err) {
|
|
2820
|
-
const errMsg = getErrorMessage(err) || '';
|
|
2821
|
-
if (isOrderDoesNotExistError(errMsg, order.orderId)) {
|
|
2822
2964
|
syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
|
|
2823
|
-
bot._log(`[DUST]
|
|
2965
|
+
bot._log(`[DUST] Cancelled ${order.id} (${order.orderId}) size=${order.size}`, 'debug');
|
|
2824
2966
|
}
|
|
2825
|
-
|
|
2826
|
-
|
|
2967
|
+
catch (err) {
|
|
2968
|
+
const errMsg = getErrorMessage(err) || '';
|
|
2969
|
+
if (isOrderDoesNotExistError(errMsg, order.orderId)) {
|
|
2970
|
+
syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
|
|
2971
|
+
bot._log(`[DUST] Order ${order.id} (${order.orderId}) already gone from chain`, 'debug');
|
|
2972
|
+
}
|
|
2973
|
+
else {
|
|
2974
|
+
bot._warn(`[DUST] Failed to cancel ${order.id} (${order.orderId}): ${errMsg}`);
|
|
2975
|
+
}
|
|
2827
2976
|
}
|
|
2828
2977
|
}
|
|
2978
|
+
return true;
|
|
2979
|
+
}, { timeoutMs: 10000, isShuttingDown: () => bot._shuttingDown === true });
|
|
2980
|
+
if (placementOk !== true) {
|
|
2981
|
+
bot._log('[DUST] Deferring dust cancels: exclusive placement region busy or shutting down', 'info');
|
|
2982
|
+
return { cancelledCount: 0, batchResult: null };
|
|
2829
2983
|
}
|
|
2830
2984
|
if (syntheticFills.length === 0)
|
|
2831
2985
|
return { cancelledCount: 0, batchResult: null };
|
|
@@ -2991,19 +3145,22 @@ async function acquireBts(bot, deficit) {
|
|
|
2991
3145
|
const sellInt = floatToBlockchainInt(best.sellAmount, best.asset.precision);
|
|
2992
3146
|
const minReceiveInt = floatToBlockchainInt(minReceive, BTS_PRECISION);
|
|
2993
3147
|
const op = chainOrders.buildLiquidityPoolExchangeOp(bot.accountId, best.poolId, sellInt, best.asset.id, minReceiveInt, coreAssetId);
|
|
2994
|
-
|
|
2995
|
-
|
|
3148
|
+
const swapOk = await runWithPlacement(bot, 'bts-acquisition-swap', async () => {
|
|
3149
|
+
try {
|
|
3150
|
+
if (!bot.privateKey) {
|
|
3151
|
+
bot._log('[BTS-ACQ] CRITICAL: No signing method available', 'error');
|
|
3152
|
+
return false;
|
|
3153
|
+
}
|
|
2996
3154
|
await chainOrders.executeBatch(bot.account, bot.privateKey, [op]);
|
|
3155
|
+
return true;
|
|
2997
3156
|
}
|
|
2998
|
-
|
|
2999
|
-
bot._log(
|
|
3000
|
-
return;
|
|
3157
|
+
catch (err) {
|
|
3158
|
+
bot._log(`[BTS-ACQ] Swap broadcast failed: ${getErrorMessage(err)}`, 'error');
|
|
3159
|
+
return false;
|
|
3001
3160
|
}
|
|
3002
|
-
}
|
|
3003
|
-
|
|
3004
|
-
bot._log(`[BTS-ACQ] Swap broadcast failed: ${getErrorMessage(err)}`, 'error');
|
|
3161
|
+
}, { timeoutMs: 10000, isShuttingDown: () => bot._shuttingDown === true });
|
|
3162
|
+
if (swapOk !== true)
|
|
3005
3163
|
return;
|
|
3006
|
-
}
|
|
3007
3164
|
const orderType = (best.asset.id === bot.assets?.assetA?.id) ? 'sell' : 'buy';
|
|
3008
3165
|
if (bot.manager.accountant) {
|
|
3009
3166
|
await bot.manager.accountant.adjustTotalBalance(orderType, -best.sellAmount, 'bts-acquisition-swap-sell');
|
|
@@ -3128,14 +3285,12 @@ async function requestGridReset(bot, reason = 'structural change', options = {})
|
|
|
3128
3285
|
// entirely (no concurrent mutation race) instead of racing them behind a
|
|
3129
3286
|
// lock.
|
|
3130
3287
|
if (options.skipFillLock === true || !bot.manager._fillProcessingLock || bot.manager._fillProcessingLock.isReentrant()) {
|
|
3131
|
-
|
|
3288
|
+
raiseRecoveryInFlight(bot);
|
|
3132
3289
|
try {
|
|
3133
3290
|
return await performGridResync(bot, resetOptions);
|
|
3134
3291
|
}
|
|
3135
3292
|
finally {
|
|
3136
|
-
|
|
3137
|
-
if ((bot._recoverySyncInFlight || 0) === 0)
|
|
3138
|
-
drainFillQueueAfterPipelineClear(bot);
|
|
3293
|
+
lowerRecoveryInFlight(bot);
|
|
3139
3294
|
}
|
|
3140
3295
|
}
|
|
3141
3296
|
return bot.manager._fillProcessingLock.acquire(async () => performGridResync(bot, resetOptions));
|
|
@@ -3214,13 +3369,13 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
3214
3369
|
// prevents a fill arriving mid-reload from starting a new COW
|
|
3215
3370
|
// batch whose broadcast would overlap the reload and force an
|
|
3216
3371
|
// avoidable commit refusal + re-adoption cycle.
|
|
3217
|
-
|
|
3372
|
+
raiseRecoveryInFlight(bot);
|
|
3218
3373
|
let persistedResult;
|
|
3219
3374
|
try {
|
|
3220
3375
|
persistedResult = await bot._recoverFromPersistedGrid();
|
|
3221
3376
|
}
|
|
3222
3377
|
finally {
|
|
3223
|
-
|
|
3378
|
+
lowerRecoveryInFlight(bot);
|
|
3224
3379
|
}
|
|
3225
3380
|
if (persistedResult.success) {
|
|
3226
3381
|
if (bot.manager?._recoveryState) {
|
|
@@ -3245,7 +3400,7 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
3245
3400
|
// generation orphans). _recoverFromPersistedGrid already
|
|
3246
3401
|
// incremented and released the counter; increment again
|
|
3247
3402
|
// for the full-resync branch.
|
|
3248
|
-
|
|
3403
|
+
raiseRecoveryInFlight(bot);
|
|
3249
3404
|
let resetResult;
|
|
3250
3405
|
try {
|
|
3251
3406
|
resetResult = await (bot.requestGridReset('rms_structural_grid_resync', {
|
|
@@ -3258,9 +3413,7 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
3258
3413
|
}));
|
|
3259
3414
|
}
|
|
3260
3415
|
finally {
|
|
3261
|
-
|
|
3262
|
-
if ((bot._recoverySyncInFlight || 0) === 0)
|
|
3263
|
-
drainFillQueueAfterPipelineClear(bot);
|
|
3416
|
+
lowerRecoveryInFlight(bot);
|
|
3264
3417
|
}
|
|
3265
3418
|
if (resetResult && bot.manager?._recoveryState) {
|
|
3266
3419
|
bot.manager._recoveryState = { ...bot.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
|
|
@@ -579,27 +579,39 @@ async function finishStartupSequence(bot, startupState) {
|
|
|
579
579
|
if (shouldRegenerate) {
|
|
580
580
|
if (!chainReadTruncated && Array.isArray(chainOpenOrders) && chainOpenOrders.length > 0) {
|
|
581
581
|
bot._log('Generating new grid and syncing with existing on-chain orders...');
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
//
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
582
|
+
// Rebuild owns placement until reconcile finishes: a fill-driven
|
|
583
|
+
// COW that starts inside this window must defer instead of
|
|
584
|
+
// re-planning against the order-id-less master.
|
|
585
|
+
bot.manager.beginStructuralRebuild?.('startup regenerate');
|
|
586
|
+
try {
|
|
587
|
+
await botGridModule(bot).initializeGrid(bot.manager);
|
|
588
|
+
await bot.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
|
|
589
|
+
// Drain queued fills BEFORE re-placement (P3 ordering): fills
|
|
590
|
+
// replayed from history reference the PRE-restart order ids.
|
|
591
|
+
// Processing them now (freshly synced grid) keeps their slot
|
|
592
|
+
// provenance; processing them after reconcile re-placement
|
|
593
|
+
// finds no grid order and falls back to orphan proceeds
|
|
594
|
+
// credits, which drift the fund bookkeeping.
|
|
595
|
+
if (bot._incomingFillQueue.length > 0) {
|
|
596
|
+
bot._log(`[STARTUP] Processing ${bot._incomingFillQueue.length} queued fill(s) before re-placement (order provenance intact)`, 'info');
|
|
597
|
+
await bot._processFillsWithBootstrapMode(chainOrders);
|
|
598
|
+
}
|
|
599
|
+
const rebalanceResult = await reconcileMod.reconcileGridOrders({
|
|
600
|
+
manager: bot.manager,
|
|
601
|
+
config: bot.config,
|
|
602
|
+
account: bot.account,
|
|
603
|
+
privateKey: bot.privateKey,
|
|
604
|
+
chainOrders,
|
|
605
|
+
chainOpenOrders,
|
|
606
|
+
// This path IS the rebuild: it owns update-first
|
|
607
|
+
// placement and must not defer on its own flag.
|
|
608
|
+
options: { allowDuringRebuild: true },
|
|
609
|
+
});
|
|
610
|
+
await bot._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
|
|
611
|
+
}
|
|
612
|
+
finally {
|
|
613
|
+
bot.manager.endStructuralRebuild?.();
|
|
593
614
|
}
|
|
594
|
-
const rebalanceResult = await reconcileMod.reconcileGridOrders({
|
|
595
|
-
manager: bot.manager,
|
|
596
|
-
config: bot.config,
|
|
597
|
-
account: bot.account,
|
|
598
|
-
privateKey: bot.privateKey,
|
|
599
|
-
chainOrders,
|
|
600
|
-
chainOpenOrders,
|
|
601
|
-
});
|
|
602
|
-
await bot._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
|
|
603
615
|
}
|
|
604
616
|
else if (!chainReadTruncated) {
|
|
605
617
|
bot._log('Generating new grid and placing initial orders on-chain...');
|
|
@@ -98,6 +98,38 @@ declare function recoverFromPersistedGrid(bot: BotLike): Promise<{
|
|
|
98
98
|
reason: string;
|
|
99
99
|
halt: boolean;
|
|
100
100
|
}>;
|
|
101
|
+
/**
|
|
102
|
+
* Detect live orders resting INSIDE the committed gap band and schedule the
|
|
103
|
+
* repair (fill-less COW re-plan + cancel-only teeth as last resort).
|
|
104
|
+
*
|
|
105
|
+
* Why this exists: a boundary commit that moves the boundary away from a live
|
|
106
|
+
* order strands it inside the spread band. The plan normally evacuates it with
|
|
107
|
+
* a rotation in the SAME batch, but the LAST-FILL-GUARD re-proves an evacuation
|
|
108
|
+
* against the PRE-COMMIT geometry and refuses it as stale when the plan's own
|
|
109
|
+
* crawl is what created the stranding (fail-closed by design — see the
|
|
110
|
+
* 2026-10-30 bid-edge incident). The batch then commits the new boundary with
|
|
111
|
+
* the stranded order still live, and because plans are fill-driven the state
|
|
112
|
+
* never heals while the market is quiet: the 4-slot spread band keeps two live
|
|
113
|
+
* buys and the real spread collapses (observed: 1.21% against a 2.0% target).
|
|
114
|
+
*
|
|
115
|
+
* Re-planning AFTER the commit is safe where the in-batch rotation is not: the
|
|
116
|
+
* committed geometry is now the LIVE geometry, so the evacuation stamp passes
|
|
117
|
+
* `isEvacuationStampStillValid` and the rotation executes as one price update.
|
|
118
|
+
*
|
|
119
|
+
* Cheap when healthy: one `detectGapEvacuationCandidates` scan (O(grid)) and an
|
|
120
|
+
* early return. Callers run it on commit (writer of the stranding) and on the
|
|
121
|
+
* maintenance tick (covers restarts and commits from paths without a bot
|
|
122
|
+
* reference).
|
|
123
|
+
*
|
|
124
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
125
|
+
* @param {string} context - Label for logging/telemetry
|
|
126
|
+
* @returns `{ stranded, scheduled }` — stranded slot ids found, whether a
|
|
127
|
+
* re-plan was scheduled.
|
|
128
|
+
*/
|
|
129
|
+
declare function scheduleGapEvacRepair(bot: BotLike, context?: string): {
|
|
130
|
+
stranded: string[];
|
|
131
|
+
scheduled: boolean;
|
|
132
|
+
};
|
|
101
133
|
/**
|
|
102
134
|
* Schedule a best-effort target-grid rebalance after a recovery sync.
|
|
103
135
|
*
|
|
@@ -126,5 +158,5 @@ declare function rejectCorruptedGridSnapshot(bot: BotLike, context: string): Pro
|
|
|
126
158
|
* @returns {Promise<boolean>}
|
|
127
159
|
*/
|
|
128
160
|
declare function targetedOrderRepair(bot: BotLike, orderIds: string[]): Promise<boolean>;
|
|
129
|
-
export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
|
|
161
|
+
export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, scheduleGapEvacRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
|
|
130
162
|
//# sourceMappingURL=dexbot_state_recovery.d.ts.map
|