dexbot 1.7.0 → 1.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +3 -3
  3. package/analysis/README.md +33 -13
  4. package/claw/package.json +1 -1
  5. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  6. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  7. package/dist/analysis/account_resolver.js +5 -6
  8. package/dist/analysis/analyze_liquidity.js +15 -4
  9. package/dist/analysis/bot_key_utils.d.ts +44 -1
  10. package/dist/analysis/bot_key_utils.js +76 -7
  11. package/dist/analysis/fills_cache.js +2 -1
  12. package/dist/analysis/grid_correction_check.d.ts +1 -19
  13. package/dist/analysis/grid_correction_check.js +12 -30
  14. package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
  15. package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
  16. package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
  17. package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
  18. package/dist/analysis/liquidity/liquidity_view.js +11 -0
  19. package/dist/analysis/resolve_bot_accounts.js +7 -6
  20. package/dist/analysis/trade_profitability.js +2 -1
  21. package/dist/credential-daemon.js +13 -2
  22. package/dist/dexbot.js +4 -4
  23. package/dist/modules/account_bots.d.ts +12 -1
  24. package/dist/modules/account_bots.js +7 -5
  25. package/dist/modules/account_orders.d.ts +12 -0
  26. package/dist/modules/account_orders.js +30 -0
  27. package/dist/modules/bitshares-native/transport.js +24 -6
  28. package/dist/modules/bitshares_client.d.ts +24 -6
  29. package/dist/modules/bitshares_client.js +44 -7
  30. package/dist/modules/bot_settings.d.ts +2 -0
  31. package/dist/modules/constants.d.ts +28 -3
  32. package/dist/modules/constants.js +79 -10
  33. package/dist/modules/dexbot_class.d.ts +23 -6
  34. package/dist/modules/dexbot_class.js +44 -0
  35. package/dist/modules/dexbot_cow_runtime.d.ts +128 -8
  36. package/dist/modules/dexbot_cow_runtime.js +688 -188
  37. package/dist/modules/dexbot_fill_runtime.js +124 -35
  38. package/dist/modules/dexbot_maintenance_runtime.d.ts +1 -1
  39. package/dist/modules/dexbot_maintenance_runtime.js +259 -79
  40. package/dist/modules/dexbot_startup_runtime.js +32 -20
  41. package/dist/modules/dexbot_state_recovery.d.ts +33 -1
  42. package/dist/modules/dexbot_state_recovery.js +167 -5
  43. package/dist/modules/node_connect_policy.d.ts +8 -1
  44. package/dist/modules/node_connect_policy.js +18 -2
  45. package/dist/modules/node_flap_score.d.ts +87 -0
  46. package/dist/modules/node_flap_score.js +124 -0
  47. package/dist/modules/node_health_cache.d.ts +40 -3
  48. package/dist/modules/node_health_cache.js +66 -20
  49. package/dist/modules/node_manager.d.ts +101 -19
  50. package/dist/modules/node_manager.js +255 -65
  51. package/dist/modules/order/accounting.d.ts +9 -4
  52. package/dist/modules/order/accounting.js +131 -19
  53. package/dist/modules/order/async_lock.d.ts +7 -1
  54. package/dist/modules/order/async_lock.js +7 -1
  55. package/dist/modules/order/export.js +25 -18
  56. package/dist/modules/order/fill_ledger.d.ts +99 -0
  57. package/dist/modules/order/fill_ledger.js +103 -0
  58. package/dist/modules/order/format.d.ts +24 -10
  59. package/dist/modules/order/format.js +30 -10
  60. package/dist/modules/order/grid.d.ts +75 -18
  61. package/dist/modules/order/grid.js +289 -72
  62. package/dist/modules/order/grid_reconcile.d.ts +5 -1
  63. package/dist/modules/order/grid_reconcile.js +333 -210
  64. package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
  65. package/dist/modules/order/grid_reconcile_internal.js +223 -91
  66. package/dist/modules/order/index.d.ts +18 -5
  67. package/dist/modules/order/index.js +5 -5
  68. package/dist/modules/order/lifecycle.d.ts +353 -0
  69. package/dist/modules/order/lifecycle.js +1109 -0
  70. package/dist/modules/order/manager.d.ts +277 -20
  71. package/dist/modules/order/manager.js +680 -64
  72. package/dist/modules/order/placement_lock.d.ts +102 -0
  73. package/dist/modules/order/placement_lock.js +222 -0
  74. package/dist/modules/order/strategy.js +13 -1
  75. package/dist/modules/order/sync_engine.d.ts +74 -114
  76. package/dist/modules/order/sync_engine.js +254 -198
  77. package/dist/modules/order/utils/math.d.ts +29 -4
  78. package/dist/modules/order/utils/math.js +61 -4
  79. package/dist/modules/order/utils/order.d.ts +106 -37
  80. package/dist/modules/order/utils/order.js +256 -81
  81. package/dist/modules/order/utils/system.d.ts +99 -6
  82. package/dist/modules/order/utils/system.js +249 -35
  83. package/dist/modules/order/utils/validate.d.ts +7 -1
  84. package/dist/modules/order/utils/validate.js +48 -2
  85. package/dist/modules/outcome.d.ts +54 -0
  86. package/dist/modules/outcome.js +51 -0
  87. package/dist/modules/settings_merge.d.ts +4 -0
  88. package/dist/modules/settings_merge.js +9 -0
  89. package/dist/modules/timing_policy.d.ts +82 -0
  90. package/dist/modules/timing_policy.js +244 -0
  91. package/dist/modules/types.d.ts +149 -22
  92. package/dist/scripts/analyze-credit.js +54 -17
  93. package/dist/scripts/analyze-orders.d.ts +4 -2
  94. package/dist/scripts/analyze-orders.js +113 -34
  95. package/dist/scripts/chart_targets.js +11 -15
  96. package/dist/scripts/test-credit-renewal.js +12 -5
  97. package/docs/COW_INVARIANTS.md +71 -8
  98. package/docs/DEXBOT_COMPARISON.md +6 -6
  99. package/docs/EVOLUTION.md +10 -8
  100. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  101. package/docs/GRID_PRICE_INVARIANT.md +1 -1
  102. package/docs/GRID_RECONCILE.md +18 -13
  103. package/docs/LOGGING.md +25 -1
  104. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +183 -26
  105. package/docs/README.md +23 -2
  106. package/docs/WORKFLOW.md +9 -3
  107. package/docs/architecture.md +68 -13
  108. package/docs/developer_guide.md +25 -8
  109. package/modules/README.md +6 -0
  110. package/package.json +3 -3
  111. package/scripts/README.md +16 -0
  112. package/scripts/pdev.sh +14 -1
  113. package/scripts/pmain.sh +23 -8
  114. package/scripts/ptest.sh +7 -0
  115. package/tests/README.md +30 -1
@@ -8,7 +8,8 @@ const require = createRequire(import.meta.url);
8
8
  import { lazyShim } from './lazy_shim.js';
9
9
  import fs from 'node:fs';
10
10
  import { spawn } from 'node:child_process';
11
- import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills } from './order/utils/system.js';
11
+ import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills, resolveGridUpdateBlockers, noteGridUpdateBlockers } from './order/utils/system.js';
12
+ import { withPlacement } from './order/placement_lock.js';
12
13
  import { path } from './path_api.js';
13
14
  import * as chainOrders from './chain_orders.js';
14
15
  import * as grid from './order/grid.js';
@@ -29,6 +30,11 @@ import { BroadcastUncertainError } from './dexbot_credential_client.js';
29
30
  import * as configModule from './config.js';
30
31
  const { Config } = configModule;
31
32
  import { getErrorMessage } from './utils/errors.js';
33
+ /**
34
+ * Gap-evac stranding sweep (lazy import: the recovery module pulls the
35
+ * bitshares client into the graph, which this runtime loads on demand already).
36
+ */
37
+ const scheduleGapEvacRepair = lazyShim('./dexbot_state_recovery', 'scheduleGapEvacRepair');
32
38
  import { isSameBotName } from './utils/sanitize_key.js';
33
39
  import { usesAmaGridPrice } from './grid_price_source.js';
34
40
  const hasOpenOrdersSyncLoopMsSet = lazyShim('./config', 'hasOpenOrdersSyncLoopMsSet');
@@ -42,6 +48,7 @@ const applyGridDivergenceCorrections = lazyShim('./order/utils/system', 'applyGr
42
48
  const updateGridFromBlockchainSnapshot = lazyShim('./order/grid', 'updateGridFromBlockchainSnapshot');
43
49
  const loadAmaCenterSnapshot = lazyShim('./order/utils/system', 'loadAmaCenterSnapshot');
44
50
  const sleep = lazyShim('./order/utils/system', 'sleep');
51
+ const getPipelineInFlightAgeMs = lazyShim('./order/utils/system', 'getPipelineInFlightAgeMs');
45
52
  const parseJsonWithComments = lazyShim('./order/utils/system', 'parseJsonWithComments');
46
53
  const isPm2Runtime = lazyShim('./order/logger', 'isPm2Runtime');
47
54
  const isWrapperAdapterOwner = lazyShim('./launcher/adapter_requirement', 'isWrapperAdapterOwner');
@@ -70,6 +77,7 @@ const isStrandedHoldOrder = lazyShim('./order/utils/order', 'isStrandedHoldOrder
70
77
  const getSideBudget = lazyShim('./order/utils/order', 'getSideBudget');
71
78
  function getCoreOrdersTotal(config) { return require('./order/utils/order').getCoreOrdersTotal(config); }
72
79
  const correctAllPriceMismatches = lazyShim('./order/utils/order', 'correctAllPriceMismatches');
80
+ const getLivePendingBroadcasts = lazyShim('./order/utils/order', 'getLivePendingBroadcasts');
73
81
  const isOrderOnChain = lazyShim('./order/utils/order', 'isOrderOnChain');
74
82
  const parseChainOrder = lazyShim('./order/utils/order', 'parseChainOrder');
75
83
  const parseSlotIndex = lazyShim('./order/utils/slot', 'parseSlotIndex');
@@ -732,8 +740,19 @@ function getTargetedSyncReason(bot) {
732
740
  }
733
741
  async function maybeRunTargetedDriftReconciliation(bot, context) {
734
742
  const trigger = getTargetedSyncReason(bot);
735
- if (!trigger)
743
+ if (!trigger) {
744
+ // CONSECUTIVE means consecutive: a tick that never evaluates the gate
745
+ // must clear the counter. Without this reset a counter parked at
746
+ // GRID_BLOCKED_ESCALATE_CYCLES survives an arbitrarily long
747
+ // trigger-less stretch, and the first unreadable cycle afterwards fires
748
+ // a structural resync instead of waiting the full window.
749
+ // (docs/FILL_PIPELINE.md B1; pinned by test_targeted_drift_reconcile.)
750
+ // Guarded: without a manager there is no counter to clear, and the
751
+ // helper would otherwise park a stray field on the bot.
752
+ if (bot.manager)
753
+ noteGridUpdateBlockers(bot, []);
736
754
  return false;
755
+ }
737
756
  // UNSETTLED-FILL GATE: a startup-style reconcile derives its window from the
738
757
  // master grid. While fills are still queued / draining / in flight / owed a
739
758
  // crawl, that master still holds the filled slots as ACTIVE and the boundary
@@ -744,9 +763,25 @@ async function maybeRunTargetedDriftReconciliation(bot, context) {
744
763
  // resolves against the crawled rail. No cooldown stamp, so the retry is
745
764
  // immediate once the fills settle.
746
765
  const unsettled = hasUnsettledFills(bot);
747
- if (unsettled.unsettled) {
748
- bot._log(`[TARGETED-SYNC] Deferring ${context} reconciliation: ${unsettled.reason} — ` +
749
- `the rail must absorb the fills before it is topped up`, 'debug');
766
+ const gridBlockers = resolveGridUpdateBlockers(bot);
767
+ // Escalation: only an UNREADABLE pipeline state escalates, and only after
768
+ // GRID_BLOCKED_ESCALATE_CYCLES consecutive attempts. A busy pipeline is
769
+ // normal work; the placement/broadcast watchdogs own their own bounds.
770
+ const blockedGate = noteGridUpdateBlockers(bot, gridBlockers);
771
+ if (blockedGate.escalate) {
772
+ const codes = gridBlockers.map((b) => b.code).join(',');
773
+ bot._log(`[GRID-BLOCKED] ${blockedGate.cycles} consecutive blocked cycle(s) with unreadable pipeline state ` +
774
+ `(${codes}); requesting a structural resync`, 'error');
775
+ fireStructuralResync(bot, 'grid update blocked by unreadable pipeline state', { cycles: blockedGate.cycles, codes }, '[GRID-BLOCKED]');
776
+ }
777
+ if (gridBlockers.length > 0) {
778
+ // Any blocker defers the reconcile: it is a chain-mutating update, and
779
+ // mutating while a fill batch, a broadcast or a placement region is
780
+ // unresolved is what built the stale-master plans this gate exists for.
781
+ const codes = gridBlockers.map((b) => b.code).join(',');
782
+ const reason = unsettled.reason || gridBlockers[0].detail;
783
+ bot._log(`[TARGETED-SYNC] Deferring ${context} reconciliation: ${reason} ` +
784
+ `[${codes}] — the rail must absorb the fills before it is topped up`, 'debug');
750
785
  return false;
751
786
  }
752
787
  const now = Date.now();
@@ -1315,6 +1350,10 @@ function performGridResync(bot, options = {}) {
1315
1350
  return Promise.resolve(false);
1316
1351
  }
1317
1352
  self.manager.startBootstrap();
1353
+ // Set BEFORE the async work so a fill-driven COW that starts while the
1354
+ // resync reloads config/re-derives the centre already sees the rebuild and
1355
+ // defers. recalculateGrid adds a second (balanced) level for its own body.
1356
+ self.manager.beginStructuralRebuild?.('trigger-file resync');
1318
1357
  self._log('Grid regeneration triggered. Performing full grid resync...');
1319
1358
  return (async () => {
1320
1359
  try {
@@ -1436,6 +1475,7 @@ function performGridResync(bot, options = {}) {
1436
1475
  }
1437
1476
  finally {
1438
1477
  self.manager.finishBootstrap();
1478
+ self.manager.endStructuralRebuild?.();
1439
1479
  }
1440
1480
  return success;
1441
1481
  })();
@@ -2061,7 +2101,7 @@ function stopBotsConfigPollInterval(bot) {
2061
2101
  /**
2062
2102
  * Release the market adapter runtime for a bot.
2063
2103
  * In PM2 mode this is a no-op; in direct mode it calls the shared runtime's releaseBot.
2064
- * @param {import('./dexbot_class.js').DEXBot} bot
2104
+ * @param {unknown} _bot - Unused; release is keyed by botId.
2065
2105
  * @param {string} botId - Bot identifier
2066
2106
  * @param {string} [context='shutdown'] - Context label for logging
2067
2107
  * @returns {Promise<any>}
@@ -2265,6 +2305,77 @@ function scheduleDeferredGridResync(ctx, options = {}) {
2265
2305
  });
2266
2306
  }, delayMs);
2267
2307
  }
2308
+ /**
2309
+ * Raise/lower the recovery-sync in-flight counter as a pair that also
2310
+ * maintains `_recoverySyncInFlightStartedAt`. The leak watchdog keys off the
2311
+ * timestamp: incrementing without it makes the watchdog skip the window (or
2312
+ * use a stale timestamp and force-clear a live recovery).
2313
+ * @param {import('./dexbot_class.js').DEXBot} bot
2314
+ */
2315
+ function raiseRecoveryInFlight(bot) {
2316
+ bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
2317
+ bot._recoverySyncInFlightStartedAt = Date.now();
2318
+ }
2319
+ /** @param {import('./dexbot_class.js').DEXBot} bot */
2320
+ function lowerRecoveryInFlight(bot) {
2321
+ bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
2322
+ if ((bot._recoverySyncInFlight || 0) === 0) {
2323
+ bot._recoverySyncInFlightStartedAt = 0;
2324
+ drainFillQueueAfterPipelineClear(bot);
2325
+ }
2326
+ }
2327
+ /**
2328
+ * Leak watchdog for the bot-level pipeline counters. A missed decrement parks
2329
+ * the fill queue indefinitely; there is no stale-clear equivalent for
2330
+ * _batchInFlight / _recoverySyncInFlight. Surface the age every tick, and only
2331
+ * force-clear when nothing actually holds the placement region (a live batch
2332
+ * owns it, so resetting there would allow a second concurrent broadcast).
2333
+ * @param {import('./dexbot_class.js').DEXBot} bot
2334
+ */
2335
+ function checkPipelineInFlightWatchdogs(bot) {
2336
+ const now = Date.now();
2337
+ const placementHeld = typeof bot.manager?.isPlacementActive === 'function'
2338
+ && bot.manager.isPlacementActive() === true;
2339
+ const checks = [
2340
+ {
2341
+ countKey: '_batchInFlight',
2342
+ startedKey: '_batchInFlightStartedAt',
2343
+ label: 'batch',
2344
+ staleMs: Number(TIMING.BATCH_INFLIGHT_STALE_MS) > 0 ? Number(TIMING.BATCH_INFLIGHT_STALE_MS) : 10 * 60 * 1000,
2345
+ },
2346
+ {
2347
+ countKey: '_recoverySyncInFlight',
2348
+ startedKey: '_recoverySyncInFlightStartedAt',
2349
+ label: 'recovery sync',
2350
+ staleMs: Number(TIMING.RECOVERY_INFLIGHT_STALE_MS) > 0 ? Number(TIMING.RECOVERY_INFLIGHT_STALE_MS) : 10 * 60 * 1000,
2351
+ },
2352
+ ];
2353
+ const botAny = bot;
2354
+ for (const check of checks) {
2355
+ const count = Number(botAny[check.countKey]) || 0;
2356
+ if (count <= 0)
2357
+ continue;
2358
+ const startedAt = Number(botAny[check.startedKey]) || 0;
2359
+ if (!startedAt)
2360
+ continue;
2361
+ const ageMs = now - startedAt;
2362
+ if (ageMs < check.staleMs)
2363
+ continue;
2364
+ if (placementHeld) {
2365
+ bot._warn(`[WATCHDOG] ${check.label} counter (${count}) has been in flight for ${Math.round(ageMs / 1000)}s ` +
2366
+ `but the placement region is still held by ${bot.manager.placementKind?.() || 'unknown'}; not force-clearing`);
2367
+ continue;
2368
+ }
2369
+ bot._warn(`[WATCHDOG] Force-clearing leaked ${check.label} counter (${count}) after ${Math.round(ageMs / 1000)}s ` +
2370
+ `with no placement holder; the pipeline must not stay parked`);
2371
+ botAny[check.countKey] = 0;
2372
+ botAny[check.startedKey] = 0;
2373
+ // Wake the fill consumer: the deferred branch gates on this counter
2374
+ // without rescheduling, so a force-clear that does not drain would
2375
+ // leave the queue parked until an unrelated event.
2376
+ drainFillQueueAfterPipelineClear(bot);
2377
+ }
2378
+ }
2268
2379
  /**
2269
2380
  * Execute the core maintenance logic: recalculate funds, check pipeline,
2270
2381
  * refresh dynamic weights, check grid health, cancel dust orders,
@@ -2277,10 +2388,21 @@ async function executeMaintenanceLogic(bot, context) {
2277
2388
  // Surface any over-long _gridLock hold first — observational only, and it
2278
2389
  // must run even when the rest of the tick defers on a broadcast region.
2279
2390
  checkGridLockHoldDuration(bot);
2280
- // Clear stale broadcast flag first so any downstream gating on
2281
- // isBroadcastingActive() (e.g. recalculateFunds, BTS balance check)
2282
- // sees the freshest state rather than a hung flag.
2391
+ // Placement leak watchdog first: it is the authoritative mutex and the one
2392
+ // authority that may tear a hung region down (it also drops that region's
2393
+ // stale broadcast refcount atomically). A live pipeline counter younger than
2394
+ // the bound keeps the region alive, so a slow create-group reconcile is not
2395
+ // cut out from under its own leases.
2396
+ try {
2397
+ bot.manager?._clearStalePlacementRegion?.(undefined, {
2398
+ batchInFlightMs: getPipelineInFlightAgeMs(bot),
2399
+ });
2400
+ }
2401
+ catch { /* observational */ }
2402
+ // Broadcast-flag watchdog: it refuses to act while a placement lease is
2403
+ // held, so it only clears regions that never took the exclusive region.
2283
2404
  bot.manager._clearStaleBroadcastFlag();
2405
+ checkPipelineInFlightWatchdogs(bot);
2284
2406
  await bot.manager.recalculateFunds();
2285
2407
  await checkBtsBalanceAndAcquire(bot);
2286
2408
  bot.manager.clearStalePipelineOperations();
@@ -2326,9 +2448,19 @@ async function executeMaintenanceLogic(bot, context) {
2326
2448
  // with batch broadcasts — transient mismatches are expected during a COW cycle.
2327
2449
  // Also skip while a structural recovery reload is in flight (it runs without
2328
2450
  // the fill lock, so this check must gate on the recovery flag explicitly).
2451
+ // Liveness count from the single authority; a test double that does not
2452
+ // model it falls back to the raw map size (a COUNT only — never a second
2453
+ // TTL policy), so a partial mock cannot make the gate throw.
2454
+ let pendingBroadcastCount = 0;
2455
+ try {
2456
+ pendingBroadcastCount = Number(getLivePendingBroadcasts(bot.manager)?.length) || 0;
2457
+ }
2458
+ catch {
2459
+ pendingBroadcastCount = Number(bot.manager?._pendingBroadcasts?.size) || 0;
2460
+ }
2329
2461
  if (!bot._recoverySyncInFlight &&
2330
2462
  !bot._batchInFlight &&
2331
- !(bot.manager?._pendingBroadcasts?.size > 0) &&
2463
+ pendingBroadcastCount === 0 &&
2332
2464
  (bot._lightweightSyncCheckAt == null || Date.now() - bot._lightweightSyncCheckAt >= TIMING.LIGHTWEIGHT_SYNC_CHECK_INTERVAL_MS)) {
2333
2465
  bot._lightweightSyncCheckAt = Date.now();
2334
2466
  try {
@@ -2368,6 +2500,22 @@ async function executeMaintenanceLogic(bot, context) {
2368
2500
  bot._log(`[LIGHTWEIGHT-SYNC] Check failed: ${getErrorMessage(e)}`, 'debug');
2369
2501
  }
2370
2502
  }
2503
+ // GAP-EVAC stranding sweep: a boundary commit that moved away from a live
2504
+ // order strands it inside the spread band, and plans are fill-driven — so a
2505
+ // quiet market never heals it (the band keeps live orders and the real
2506
+ // spread stays collapsed). The sweep re-derives the streak ledger against
2507
+ // the COMMITTED geometry, queues the cancel-only teeth for anything past
2508
+ // the threshold, and schedules a fill-less re-plan (the preferred repair:
2509
+ // one in-place price update). It early-returns on a healthy band, so it is
2510
+ // cheap enough to run every tick, and it also covers commits made by paths
2511
+ // that hold no bot reference (divergence/sync) plus a restart that loads an
2512
+ // already-stranded snapshot.
2513
+ try {
2514
+ scheduleGapEvacRepair(bot, `maintenance ${context}`);
2515
+ }
2516
+ catch (gapEvacErr) {
2517
+ bot._log(`[GAP-EVAC] Stranding sweep failed (non-fatal): ${getErrorMessage(gapEvacErr)}`, 'debug');
2518
+ }
2371
2519
  // Process any price corrections queued by prior sync operations before
2372
2520
  // the pipeline gate. Pending corrections block isPipelineEmpty, and no
2373
2521
  // other code path clears them outside of _consumeFillQueue (which only
@@ -2375,20 +2523,34 @@ async function executeMaintenanceLogic(bot, context) {
2375
2523
  // during startup or periodic sync can stall the pipeline indefinitely.
2376
2524
  const pendingCorrections = bot.manager.ordersNeedingPriceCorrection?.length || 0;
2377
2525
  if (pendingCorrections > 0) {
2378
- const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders);
2379
- if (correctionResult.failed > 0) {
2380
- const failedDetails = (correctionResult.results || [])
2381
- .filter((r) => !(r.result && r.result.success))
2382
- .slice(0, 3)
2383
- .map((r) => `${r.chainOrderId ?? '?'}: ${r.result?.error ?? 'unknown'}`)
2384
- .join(' | ');
2385
- bot._warn(`[MAINT] ${correctionResult.failed}/${pendingCorrections} price correction(s) failed` +
2386
- (failedDetails ? ` — ${failedDetails}` : '') +
2387
- (correctionResult.staleDropped > 0 ? ` (${correctionResult.staleDropped} stale dropped)` : ''));
2388
- }
2389
- else if (correctionResult.staleDropped > 0) {
2390
- bot._log(`[MAINT] ${correctionResult.corrected}/${pendingCorrections} price correction(s) resolved, ` +
2391
- `${correctionResult.staleDropped} stale dropped (resync moved the slot; re-queued by next sync if still off-target)`, 'info');
2526
+ // The drain cancels/updates chain orders directly, so it must own the
2527
+ // exclusive placement region. `correctAllPriceMismatches` acquires it
2528
+ // internally (single entry point); this call uses a short bound because
2529
+ // this is a background repair and must not block the maintenance tick
2530
+ // behind a long broadcast. A busy region leaves the queue for the next
2531
+ // tick (or the region-end wake-up).
2532
+ try {
2533
+ const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders, { timeoutMs: 250 });
2534
+ if (correctionResult?.deferred === true) {
2535
+ bot._log('[MAINT] Deferring price-correction drain: exclusive placement region busy or shutting down', 'info');
2536
+ }
2537
+ else if (correctionResult.failed > 0) {
2538
+ const failedDetails = (correctionResult.results || [])
2539
+ .filter((r) => !(r.result && r.result.success))
2540
+ .slice(0, 3)
2541
+ .map((r) => `${r.chainOrderId ?? '?'}: ${r.result?.error ?? 'unknown'}`)
2542
+ .join(' | ');
2543
+ bot._warn(`[MAINT] ${correctionResult.failed}/${pendingCorrections} price correction(s) failed` +
2544
+ (failedDetails ? ` — ${failedDetails}` : '') +
2545
+ (correctionResult.staleDropped > 0 ? ` (${correctionResult.staleDropped} stale dropped)` : ''));
2546
+ }
2547
+ else if (correctionResult.staleDropped > 0) {
2548
+ bot._log(`[MAINT] ${correctionResult.corrected}/${pendingCorrections} price correction(s) resolved, ` +
2549
+ `${correctionResult.staleDropped} stale dropped (resync moved the slot; re-queued by next sync if still off-target)`, 'info');
2550
+ }
2551
+ }
2552
+ catch (correctionErr) {
2553
+ bot._warn(`[MAINT] Price-correction drain failed: ${getErrorMessage(correctionErr)}`);
2392
2554
  }
2393
2555
  }
2394
2556
  // Dust detection runs before the pipeline gate. Cancellation is immediate
@@ -2764,6 +2926,14 @@ async function cancelOrderDeferredOnUncertain(bot, order) {
2764
2926
  throw err;
2765
2927
  }
2766
2928
  }
2929
+ async function runWithPlacement(bot, kind, fn, options = {}) {
2930
+ // Bot-first adapter over the ONE acquire/release contract
2931
+ // (modules/order/placement_lock.ts): it owns the bounded wait, the shutdown
2932
+ // probe, the NOOP fallback for manager-less fakes and the release. Do not
2933
+ // re-implement that here — a local copy already drifted once (it could not
2934
+ // take a lease and skipped the lease-liveness check).
2935
+ return await withPlacement(bot.manager, kind, fn, options);
2936
+ }
2767
2937
  /**
2768
2938
  * Cancel dust orders immediately — no delay, no timer, no maps.
2769
2939
  * Each dust order is cancelled on chain and its slot is rotated through
@@ -2780,52 +2950,63 @@ async function cancelDustOrders(bot, { buy: buyDust = [], sell: sellDust = [] }
2780
2950
  if (allDust.length === 0)
2781
2951
  return { cancelledCount: 0, batchResult: null };
2782
2952
  const syntheticFills = [];
2783
- for (const order of allDust) {
2784
- if (!order.orderId)
2785
- continue;
2786
- try {
2787
- const cancelResult = await cancelOrderDeferredOnUncertain(bot, order);
2953
+ // Hold the exclusive placement region across the whole cancel pass: each
2954
+ // cancel mutates the chain and the following synchronizeWithChain mutates
2955
+ // the master. A COW batch / reconcile must not interleave between them.
2956
+ // The rotation batch below runs after release (it acquires its own lease).
2957
+ const placementOk = await runWithPlacement(bot, 'dust-cancel', async () => {
2958
+ for (const order of allDust) {
2959
+ if (!order.orderId)
2960
+ continue;
2788
2961
  try {
2789
- if (cancelResult?.verifiedAfterFailure) {
2790
- const accountRef = bot.accountId || bot.account;
2791
- // The cancel was verified absent on an authoritative
2792
- // (non-empty, non-truncated) read inside cancelOrder, so an
2793
- // ambiguous refetch must not run the full sync: the snapshot
2794
- // omits the freshest orders and pass-1 phantom cleanup could
2795
- // virtualize live slots. Fall back to the local cancel sync.
2796
- const freshOrders = await readOpenOrdersGuarded(chainOrders, accountRef, {
2797
- log: (message) => bot._warn(message),
2798
- label: 'DUST',
2799
- deferEmpty: true,
2800
- skipMessage: (kind) => `[DUST] Chain refetch after verified cancel is ${kind}; applying local cancel sync for ${order.id}`,
2801
- });
2802
- if (freshOrders === null) {
2803
- await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2962
+ const cancelResult = await cancelOrderDeferredOnUncertain(bot, order);
2963
+ try {
2964
+ if (cancelResult?.verifiedAfterFailure) {
2965
+ const accountRef = bot.accountId || bot.account;
2966
+ // The cancel was verified absent on an authoritative
2967
+ // (non-empty, non-truncated) read inside cancelOrder, so an
2968
+ // ambiguous refetch must not run the full sync: the snapshot
2969
+ // omits the freshest orders and pass-1 phantom cleanup could
2970
+ // virtualize live slots. Fall back to the local cancel sync.
2971
+ const freshOrders = await readOpenOrdersGuarded(chainOrders, accountRef, {
2972
+ log: (message) => bot._warn(message),
2973
+ label: 'DUST',
2974
+ deferEmpty: true,
2975
+ skipMessage: (kind) => `[DUST] Chain refetch after verified cancel is ${kind}; applying local cancel sync for ${order.id}`,
2976
+ });
2977
+ if (freshOrders === null) {
2978
+ await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2979
+ }
2980
+ else {
2981
+ await bot.manager.synchronizeWithChain(freshOrders, 'readOpenOrders');
2982
+ }
2804
2983
  }
2805
2984
  else {
2806
- await bot.manager.synchronizeWithChain(freshOrders, 'readOpenOrders');
2985
+ await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2807
2986
  }
2808
2987
  }
2809
- else {
2810
- await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2988
+ catch (refetchErr) {
2989
+ bot._warn(`[DUST] Cancel succeeded but refetch failed for ${order.id} (${order.orderId}): ${getErrorMessage(refetchErr)}`);
2811
2990
  }
2812
- }
2813
- catch (refetchErr) {
2814
- bot._warn(`[DUST] Cancel succeeded but refetch failed for ${order.id} (${order.orderId}): ${getErrorMessage(refetchErr)}`);
2815
- }
2816
- syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
2817
- bot._log(`[DUST] Cancelled ${order.id} (${order.orderId}) size=${order.size}`, 'debug');
2818
- }
2819
- catch (err) {
2820
- const errMsg = getErrorMessage(err) || '';
2821
- if (isOrderDoesNotExistError(errMsg, order.orderId)) {
2822
2991
  syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
2823
- bot._log(`[DUST] Order ${order.id} (${order.orderId}) already gone from chain`, 'debug');
2992
+ bot._log(`[DUST] Cancelled ${order.id} (${order.orderId}) size=${order.size}`, 'debug');
2824
2993
  }
2825
- else {
2826
- bot._warn(`[DUST] Failed to cancel ${order.id} (${order.orderId}): ${errMsg}`);
2994
+ catch (err) {
2995
+ const errMsg = getErrorMessage(err) || '';
2996
+ if (isOrderDoesNotExistError(errMsg, order.orderId)) {
2997
+ syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
2998
+ bot._log(`[DUST] Order ${order.id} (${order.orderId}) already gone from chain`, 'debug');
2999
+ }
3000
+ else {
3001
+ bot._warn(`[DUST] Failed to cancel ${order.id} (${order.orderId}): ${errMsg}`);
3002
+ }
2827
3003
  }
2828
3004
  }
3005
+ return true;
3006
+ }, { timeoutMs: 10000, isShuttingDown: () => bot._shuttingDown === true });
3007
+ if (placementOk !== true) {
3008
+ bot._log('[DUST] Deferring dust cancels: exclusive placement region busy or shutting down', 'info');
3009
+ return { cancelledCount: 0, batchResult: null };
2829
3010
  }
2830
3011
  if (syntheticFills.length === 0)
2831
3012
  return { cancelledCount: 0, batchResult: null };
@@ -2991,19 +3172,22 @@ async function acquireBts(bot, deficit) {
2991
3172
  const sellInt = floatToBlockchainInt(best.sellAmount, best.asset.precision);
2992
3173
  const minReceiveInt = floatToBlockchainInt(minReceive, BTS_PRECISION);
2993
3174
  const op = chainOrders.buildLiquidityPoolExchangeOp(bot.accountId, best.poolId, sellInt, best.asset.id, minReceiveInt, coreAssetId);
2994
- try {
2995
- if (bot.privateKey) {
3175
+ const swapOk = await runWithPlacement(bot, 'bts-acquisition-swap', async () => {
3176
+ try {
3177
+ if (!bot.privateKey) {
3178
+ bot._log('[BTS-ACQ] CRITICAL: No signing method available', 'error');
3179
+ return false;
3180
+ }
2996
3181
  await chainOrders.executeBatch(bot.account, bot.privateKey, [op]);
3182
+ return true;
2997
3183
  }
2998
- else {
2999
- bot._log('[BTS-ACQ] CRITICAL: No signing method available', 'error');
3000
- return;
3184
+ catch (err) {
3185
+ bot._log(`[BTS-ACQ] Swap broadcast failed: ${getErrorMessage(err)}`, 'error');
3186
+ return false;
3001
3187
  }
3002
- }
3003
- catch (err) {
3004
- bot._log(`[BTS-ACQ] Swap broadcast failed: ${getErrorMessage(err)}`, 'error');
3188
+ }, { timeoutMs: 10000, isShuttingDown: () => bot._shuttingDown === true });
3189
+ if (swapOk !== true)
3005
3190
  return;
3006
- }
3007
3191
  const orderType = (best.asset.id === bot.assets?.assetA?.id) ? 'sell' : 'buy';
3008
3192
  if (bot.manager.accountant) {
3009
3193
  await bot.manager.accountant.adjustTotalBalance(orderType, -best.sellAmount, 'bts-acquisition-swap-sell');
@@ -3128,14 +3312,12 @@ async function requestGridReset(bot, reason = 'structural change', options = {})
3128
3312
  // entirely (no concurrent mutation race) instead of racing them behind a
3129
3313
  // lock.
3130
3314
  if (options.skipFillLock === true || !bot.manager._fillProcessingLock || bot.manager._fillProcessingLock.isReentrant()) {
3131
- bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
3315
+ raiseRecoveryInFlight(bot);
3132
3316
  try {
3133
3317
  return await performGridResync(bot, resetOptions);
3134
3318
  }
3135
3319
  finally {
3136
- bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
3137
- if ((bot._recoverySyncInFlight || 0) === 0)
3138
- drainFillQueueAfterPipelineClear(bot);
3320
+ lowerRecoveryInFlight(bot);
3139
3321
  }
3140
3322
  }
3141
3323
  return bot.manager._fillProcessingLock.acquire(async () => performGridResync(bot, resetOptions));
@@ -3214,13 +3396,13 @@ function wireStructuralGridResyncRequest(bot) {
3214
3396
  // prevents a fill arriving mid-reload from starting a new COW
3215
3397
  // batch whose broadcast would overlap the reload and force an
3216
3398
  // avoidable commit refusal + re-adoption cycle.
3217
- bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
3399
+ raiseRecoveryInFlight(bot);
3218
3400
  let persistedResult;
3219
3401
  try {
3220
3402
  persistedResult = await bot._recoverFromPersistedGrid();
3221
3403
  }
3222
3404
  finally {
3223
- bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
3405
+ lowerRecoveryInFlight(bot);
3224
3406
  }
3225
3407
  if (persistedResult.success) {
3226
3408
  if (bot.manager?._recoveryState) {
@@ -3245,7 +3427,7 @@ function wireStructuralGridResyncRequest(bot) {
3245
3427
  // generation orphans). _recoverFromPersistedGrid already
3246
3428
  // incremented and released the counter; increment again
3247
3429
  // for the full-resync branch.
3248
- bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
3430
+ raiseRecoveryInFlight(bot);
3249
3431
  let resetResult;
3250
3432
  try {
3251
3433
  resetResult = await (bot.requestGridReset('rms_structural_grid_resync', {
@@ -3258,9 +3440,7 @@ function wireStructuralGridResyncRequest(bot) {
3258
3440
  }));
3259
3441
  }
3260
3442
  finally {
3261
- bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
3262
- if ((bot._recoverySyncInFlight || 0) === 0)
3263
- drainFillQueueAfterPipelineClear(bot);
3443
+ lowerRecoveryInFlight(bot);
3264
3444
  }
3265
3445
  if (resetResult && bot.manager?._recoveryState) {
3266
3446
  bot.manager._recoveryState = { ...bot.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
@@ -579,27 +579,39 @@ async function finishStartupSequence(bot, startupState) {
579
579
  if (shouldRegenerate) {
580
580
  if (!chainReadTruncated && Array.isArray(chainOpenOrders) && chainOpenOrders.length > 0) {
581
581
  bot._log('Generating new grid and syncing with existing on-chain orders...');
582
- await botGridModule(bot).initializeGrid(bot.manager);
583
- await bot.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
584
- // Drain queued fills BEFORE re-placement (P3 ordering): fills
585
- // replayed from history reference the PRE-restart order ids.
586
- // Processing them now (freshly synced grid) keeps their slot
587
- // provenance; processing them after reconcile re-placement
588
- // finds no grid order and falls back to orphan proceeds
589
- // credits, which drift the fund bookkeeping.
590
- if (bot._incomingFillQueue.length > 0) {
591
- bot._log(`[STARTUP] Processing ${bot._incomingFillQueue.length} queued fill(s) before re-placement (order provenance intact)`, 'info');
592
- await bot._processFillsWithBootstrapMode(chainOrders);
582
+ // Rebuild owns placement until reconcile finishes: a fill-driven
583
+ // COW that starts inside this window must defer instead of
584
+ // re-planning against the order-id-less master.
585
+ bot.manager.beginStructuralRebuild?.('startup regenerate');
586
+ try {
587
+ await botGridModule(bot).initializeGrid(bot.manager);
588
+ await bot.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
589
+ // Drain queued fills BEFORE re-placement (P3 ordering): fills
590
+ // replayed from history reference the PRE-restart order ids.
591
+ // Processing them now (freshly synced grid) keeps their slot
592
+ // provenance; processing them after reconcile re-placement
593
+ // finds no grid order and falls back to orphan proceeds
594
+ // credits, which drift the fund bookkeeping.
595
+ if (bot._incomingFillQueue.length > 0) {
596
+ bot._log(`[STARTUP] Processing ${bot._incomingFillQueue.length} queued fill(s) before re-placement (order provenance intact)`, 'info');
597
+ await bot._processFillsWithBootstrapMode(chainOrders);
598
+ }
599
+ const rebalanceResult = await reconcileMod.reconcileGridOrders({
600
+ manager: bot.manager,
601
+ config: bot.config,
602
+ account: bot.account,
603
+ privateKey: bot.privateKey,
604
+ chainOrders,
605
+ chainOpenOrders,
606
+ // This path IS the rebuild: it owns update-first
607
+ // placement and must not defer on its own flag.
608
+ options: { allowDuringRebuild: true },
609
+ });
610
+ await bot._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
611
+ }
612
+ finally {
613
+ bot.manager.endStructuralRebuild?.();
593
614
  }
594
- const rebalanceResult = await reconcileMod.reconcileGridOrders({
595
- manager: bot.manager,
596
- config: bot.config,
597
- account: bot.account,
598
- privateKey: bot.privateKey,
599
- chainOrders,
600
- chainOpenOrders,
601
- });
602
- await bot._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
603
615
  }
604
616
  else if (!chainReadTruncated) {
605
617
  bot._log('Generating new grid and placing initial orders on-chain...');
@@ -98,6 +98,38 @@ declare function recoverFromPersistedGrid(bot: BotLike): Promise<{
98
98
  reason: string;
99
99
  halt: boolean;
100
100
  }>;
101
+ /**
102
+ * Detect live orders resting INSIDE the committed gap band and schedule the
103
+ * repair (fill-less COW re-plan + cancel-only teeth as last resort).
104
+ *
105
+ * Why this exists: a boundary commit that moves the boundary away from a live
106
+ * order strands it inside the spread band. The plan normally evacuates it with
107
+ * a rotation in the SAME batch, but the LAST-FILL-GUARD re-proves an evacuation
108
+ * against the PRE-COMMIT geometry and refuses it as stale when the plan's own
109
+ * crawl is what created the stranding (fail-closed by design — see the
110
+ * 2026-10-30 bid-edge incident). The batch then commits the new boundary with
111
+ * the stranded order still live, and because plans are fill-driven the state
112
+ * never heals while the market is quiet: the 4-slot spread band keeps two live
113
+ * buys and the real spread collapses (observed: 1.21% against a 2.0% target).
114
+ *
115
+ * Re-planning AFTER the commit is safe where the in-batch rotation is not: the
116
+ * committed geometry is now the LIVE geometry, so the evacuation stamp passes
117
+ * `isEvacuationStampStillValid` and the rotation executes as one price update.
118
+ *
119
+ * Cheap when healthy: one `detectGapEvacuationCandidates` scan (O(grid)) and an
120
+ * early return. Callers run it on commit (writer of the stranding) and on the
121
+ * maintenance tick (covers restarts and commits from paths without a bot
122
+ * reference).
123
+ *
124
+ * @param {import('./dexbot_class.js').DEXBot} bot
125
+ * @param {string} context - Label for logging/telemetry
126
+ * @returns `{ stranded, scheduled }` — stranded slot ids found, whether a
127
+ * re-plan was scheduled.
128
+ */
129
+ declare function scheduleGapEvacRepair(bot: BotLike, context?: string): {
130
+ stranded: string[];
131
+ scheduled: boolean;
132
+ };
101
133
  /**
102
134
  * Schedule a best-effort target-grid rebalance after a recovery sync.
103
135
  *
@@ -126,5 +158,5 @@ declare function rejectCorruptedGridSnapshot(bot: BotLike, context: string): Pro
126
158
  * @returns {Promise<boolean>}
127
159
  */
128
160
  declare function targetedOrderRepair(bot: BotLike, orderIds: string[]): Promise<boolean>;
129
- export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
161
+ export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, scheduleGapEvacRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
130
162
  //# sourceMappingURL=dexbot_state_recovery.d.ts.map