dexbot 1.7.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/README.md +3 -3
  3. package/analysis/README.md +33 -13
  4. package/claw/package.json +1 -1
  5. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  6. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  7. package/dist/analysis/account_resolver.js +5 -6
  8. package/dist/analysis/analyze_liquidity.js +15 -4
  9. package/dist/analysis/bot_key_utils.d.ts +44 -1
  10. package/dist/analysis/bot_key_utils.js +76 -7
  11. package/dist/analysis/fills_cache.js +2 -1
  12. package/dist/analysis/grid_correction_check.d.ts +1 -19
  13. package/dist/analysis/grid_correction_check.js +12 -30
  14. package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
  15. package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
  16. package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
  17. package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
  18. package/dist/analysis/liquidity/liquidity_view.js +11 -0
  19. package/dist/analysis/resolve_bot_accounts.js +7 -6
  20. package/dist/analysis/trade_profitability.js +2 -1
  21. package/dist/credential-daemon.js +13 -2
  22. package/dist/dexbot.js +4 -4
  23. package/dist/modules/account_bots.d.ts +11 -1
  24. package/dist/modules/account_orders.d.ts +12 -0
  25. package/dist/modules/account_orders.js +30 -0
  26. package/dist/modules/bitshares-native/transport.js +24 -6
  27. package/dist/modules/bitshares_client.d.ts +24 -6
  28. package/dist/modules/bitshares_client.js +44 -7
  29. package/dist/modules/bot_settings.d.ts +2 -0
  30. package/dist/modules/constants.d.ts +26 -3
  31. package/dist/modules/constants.js +63 -8
  32. package/dist/modules/dexbot_class.d.ts +1 -6
  33. package/dist/modules/dexbot_class.js +14 -0
  34. package/dist/modules/dexbot_cow_runtime.d.ts +97 -7
  35. package/dist/modules/dexbot_cow_runtime.js +620 -132
  36. package/dist/modules/dexbot_fill_runtime.js +124 -35
  37. package/dist/modules/dexbot_maintenance_runtime.js +226 -73
  38. package/dist/modules/dexbot_startup_runtime.js +32 -20
  39. package/dist/modules/dexbot_state_recovery.d.ts +33 -1
  40. package/dist/modules/dexbot_state_recovery.js +167 -5
  41. package/dist/modules/node_connect_policy.d.ts +8 -1
  42. package/dist/modules/node_connect_policy.js +18 -2
  43. package/dist/modules/node_flap_score.d.ts +87 -0
  44. package/dist/modules/node_flap_score.js +124 -0
  45. package/dist/modules/node_health_cache.d.ts +40 -3
  46. package/dist/modules/node_health_cache.js +66 -20
  47. package/dist/modules/node_manager.d.ts +101 -19
  48. package/dist/modules/node_manager.js +255 -65
  49. package/dist/modules/order/accounting.js +56 -7
  50. package/dist/modules/order/format.d.ts +24 -10
  51. package/dist/modules/order/format.js +30 -10
  52. package/dist/modules/order/grid.js +82 -11
  53. package/dist/modules/order/grid_reconcile.d.ts +5 -1
  54. package/dist/modules/order/grid_reconcile.js +333 -210
  55. package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
  56. package/dist/modules/order/grid_reconcile_internal.js +207 -87
  57. package/dist/modules/order/index.d.ts +4 -0
  58. package/dist/modules/order/lifecycle.d.ts +353 -0
  59. package/dist/modules/order/lifecycle.js +1109 -0
  60. package/dist/modules/order/manager.d.ts +271 -5
  61. package/dist/modules/order/manager.js +651 -48
  62. package/dist/modules/order/placement_lock.d.ts +102 -0
  63. package/dist/modules/order/placement_lock.js +222 -0
  64. package/dist/modules/order/strategy.js +13 -1
  65. package/dist/modules/order/sync_engine.d.ts +13 -1
  66. package/dist/modules/order/sync_engine.js +153 -53
  67. package/dist/modules/order/utils/math.d.ts +22 -1
  68. package/dist/modules/order/utils/math.js +54 -1
  69. package/dist/modules/order/utils/order.d.ts +75 -5
  70. package/dist/modules/order/utils/order.js +158 -9
  71. package/dist/modules/order/utils/system.d.ts +18 -0
  72. package/dist/modules/order/utils/system.js +84 -8
  73. package/dist/modules/order/utils/validate.d.ts +7 -1
  74. package/dist/modules/order/utils/validate.js +48 -2
  75. package/dist/modules/outcome.d.ts +54 -0
  76. package/dist/modules/outcome.js +51 -0
  77. package/dist/modules/settings_merge.d.ts +4 -0
  78. package/dist/modules/settings_merge.js +9 -0
  79. package/dist/modules/timing_policy.d.ts +82 -0
  80. package/dist/modules/timing_policy.js +244 -0
  81. package/dist/modules/types.d.ts +132 -0
  82. package/dist/scripts/analyze-credit.js +54 -17
  83. package/dist/scripts/analyze-orders.d.ts +4 -2
  84. package/dist/scripts/analyze-orders.js +113 -34
  85. package/dist/scripts/chart_targets.js +11 -15
  86. package/dist/scripts/test-credit-renewal.js +12 -5
  87. package/docs/COW_INVARIANTS.md +67 -8
  88. package/docs/DEXBOT_COMPARISON.md +6 -6
  89. package/docs/EVOLUTION.md +9 -8
  90. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  91. package/docs/GRID_PRICE_INVARIANT.md +1 -1
  92. package/docs/GRID_RECONCILE.md +17 -12
  93. package/docs/LOGGING.md +25 -1
  94. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +175 -26
  95. package/docs/README.md +23 -2
  96. package/docs/WORKFLOW.md +9 -3
  97. package/docs/architecture.md +42 -5
  98. package/docs/developer_guide.md +25 -8
  99. package/modules/README.md +3 -0
  100. package/package.json +1 -1
  101. package/scripts/README.md +16 -0
  102. package/scripts/pdev.sh +14 -1
  103. package/scripts/pmain.sh +23 -8
  104. package/scripts/ptest.sh +7 -0
  105. package/tests/README.md +30 -1
@@ -9,6 +9,7 @@ import { lazyShim } from './lazy_shim.js';
9
9
  import fs from 'node:fs';
10
10
  import { spawn } from 'node:child_process';
11
11
  import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills } from './order/utils/system.js';
12
+ import { withPlacement } from './order/placement_lock.js';
12
13
  import { path } from './path_api.js';
13
14
  import * as chainOrders from './chain_orders.js';
14
15
  import * as grid from './order/grid.js';
@@ -29,6 +30,11 @@ import { BroadcastUncertainError } from './dexbot_credential_client.js';
29
30
  import * as configModule from './config.js';
30
31
  const { Config } = configModule;
31
32
  import { getErrorMessage } from './utils/errors.js';
33
+ /**
34
+ * Gap-evac stranding sweep (lazy import: the recovery module pulls the
35
+ * bitshares client into the graph, which this runtime loads on demand already).
36
+ */
37
+ const scheduleGapEvacRepair = lazyShim('./dexbot_state_recovery', 'scheduleGapEvacRepair');
32
38
  import { isSameBotName } from './utils/sanitize_key.js';
33
39
  import { usesAmaGridPrice } from './grid_price_source.js';
34
40
  const hasOpenOrdersSyncLoopMsSet = lazyShim('./config', 'hasOpenOrdersSyncLoopMsSet');
@@ -42,6 +48,7 @@ const applyGridDivergenceCorrections = lazyShim('./order/utils/system', 'applyGr
42
48
  const updateGridFromBlockchainSnapshot = lazyShim('./order/grid', 'updateGridFromBlockchainSnapshot');
43
49
  const loadAmaCenterSnapshot = lazyShim('./order/utils/system', 'loadAmaCenterSnapshot');
44
50
  const sleep = lazyShim('./order/utils/system', 'sleep');
51
+ const getPipelineInFlightAgeMs = lazyShim('./order/utils/system', 'getPipelineInFlightAgeMs');
45
52
  const parseJsonWithComments = lazyShim('./order/utils/system', 'parseJsonWithComments');
46
53
  const isPm2Runtime = lazyShim('./order/logger', 'isPm2Runtime');
47
54
  const isWrapperAdapterOwner = lazyShim('./launcher/adapter_requirement', 'isWrapperAdapterOwner');
@@ -70,6 +77,7 @@ const isStrandedHoldOrder = lazyShim('./order/utils/order', 'isStrandedHoldOrder
70
77
  const getSideBudget = lazyShim('./order/utils/order', 'getSideBudget');
71
78
  function getCoreOrdersTotal(config) { return require('./order/utils/order').getCoreOrdersTotal(config); }
72
79
  const correctAllPriceMismatches = lazyShim('./order/utils/order', 'correctAllPriceMismatches');
80
+ const getLivePendingBroadcasts = lazyShim('./order/utils/order', 'getLivePendingBroadcasts');
73
81
  const isOrderOnChain = lazyShim('./order/utils/order', 'isOrderOnChain');
74
82
  const parseChainOrder = lazyShim('./order/utils/order', 'parseChainOrder');
75
83
  const parseSlotIndex = lazyShim('./order/utils/slot', 'parseSlotIndex');
@@ -1315,6 +1323,10 @@ function performGridResync(bot, options = {}) {
1315
1323
  return Promise.resolve(false);
1316
1324
  }
1317
1325
  self.manager.startBootstrap();
1326
+ // Set BEFORE the async work so a fill-driven COW that starts while the
1327
+ // resync reloads config/re-derives the centre already sees the rebuild and
1328
+ // defers. recalculateGrid adds a second (balanced) level for its own body.
1329
+ self.manager.beginStructuralRebuild?.('trigger-file resync');
1318
1330
  self._log('Grid regeneration triggered. Performing full grid resync...');
1319
1331
  return (async () => {
1320
1332
  try {
@@ -1436,6 +1448,7 @@ function performGridResync(bot, options = {}) {
1436
1448
  }
1437
1449
  finally {
1438
1450
  self.manager.finishBootstrap();
1451
+ self.manager.endStructuralRebuild?.();
1439
1452
  }
1440
1453
  return success;
1441
1454
  })();
@@ -2265,6 +2278,77 @@ function scheduleDeferredGridResync(ctx, options = {}) {
2265
2278
  });
2266
2279
  }, delayMs);
2267
2280
  }
2281
+ /**
2282
+ * Raise/lower the recovery-sync in-flight counter as a pair that also
2283
+ * maintains `_recoverySyncInFlightStartedAt`. The leak watchdog keys off the
2284
+ * timestamp: incrementing without it makes the watchdog skip the window (or
2285
+ * use a stale timestamp and force-clear a live recovery).
2286
+ * @param {import('./dexbot_class.js').DEXBot} bot
2287
+ */
2288
+ function raiseRecoveryInFlight(bot) {
2289
+ bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
2290
+ bot._recoverySyncInFlightStartedAt = Date.now();
2291
+ }
2292
+ /** @param {import('./dexbot_class.js').DEXBot} bot */
2293
+ function lowerRecoveryInFlight(bot) {
2294
+ bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
2295
+ if ((bot._recoverySyncInFlight || 0) === 0) {
2296
+ bot._recoverySyncInFlightStartedAt = 0;
2297
+ drainFillQueueAfterPipelineClear(bot);
2298
+ }
2299
+ }
2300
+ /**
2301
+ * Leak watchdog for the bot-level pipeline counters. A missed decrement parks
2302
+ * the fill queue indefinitely; there is no stale-clear equivalent for
2303
+ * _batchInFlight / _recoverySyncInFlight. Surface the age every tick, and only
2304
+ * force-clear when nothing actually holds the placement region (a live batch
2305
+ * owns it, so resetting there would allow a second concurrent broadcast).
2306
+ * @param {import('./dexbot_class.js').DEXBot} bot
2307
+ */
2308
+ function checkPipelineInFlightWatchdogs(bot) {
2309
+ const now = Date.now();
2310
+ const placementHeld = typeof bot.manager?.isPlacementActive === 'function'
2311
+ && bot.manager.isPlacementActive() === true;
2312
+ const checks = [
2313
+ {
2314
+ countKey: '_batchInFlight',
2315
+ startedKey: '_batchInFlightStartedAt',
2316
+ label: 'batch',
2317
+ staleMs: Number(TIMING.BATCH_INFLIGHT_STALE_MS) > 0 ? Number(TIMING.BATCH_INFLIGHT_STALE_MS) : 10 * 60 * 1000,
2318
+ },
2319
+ {
2320
+ countKey: '_recoverySyncInFlight',
2321
+ startedKey: '_recoverySyncInFlightStartedAt',
2322
+ label: 'recovery sync',
2323
+ staleMs: Number(TIMING.RECOVERY_INFLIGHT_STALE_MS) > 0 ? Number(TIMING.RECOVERY_INFLIGHT_STALE_MS) : 10 * 60 * 1000,
2324
+ },
2325
+ ];
2326
+ const botAny = bot;
2327
+ for (const check of checks) {
2328
+ const count = Number(botAny[check.countKey]) || 0;
2329
+ if (count <= 0)
2330
+ continue;
2331
+ const startedAt = Number(botAny[check.startedKey]) || 0;
2332
+ if (!startedAt)
2333
+ continue;
2334
+ const ageMs = now - startedAt;
2335
+ if (ageMs < check.staleMs)
2336
+ continue;
2337
+ if (placementHeld) {
2338
+ bot._warn(`[WATCHDOG] ${check.label} counter (${count}) has been in flight for ${Math.round(ageMs / 1000)}s ` +
2339
+ `but the placement region is still held by ${bot.manager.placementKind?.() || 'unknown'}; not force-clearing`);
2340
+ continue;
2341
+ }
2342
+ bot._warn(`[WATCHDOG] Force-clearing leaked ${check.label} counter (${count}) after ${Math.round(ageMs / 1000)}s ` +
2343
+ `with no placement holder; the pipeline must not stay parked`);
2344
+ botAny[check.countKey] = 0;
2345
+ botAny[check.startedKey] = 0;
2346
+ // Wake the fill consumer: the deferred branch gates on this counter
2347
+ // without rescheduling, so a force-clear that does not drain would
2348
+ // leave the queue parked until an unrelated event.
2349
+ drainFillQueueAfterPipelineClear(bot);
2350
+ }
2351
+ }
2268
2352
  /**
2269
2353
  * Execute the core maintenance logic: recalculate funds, check pipeline,
2270
2354
  * refresh dynamic weights, check grid health, cancel dust orders,
@@ -2277,10 +2361,21 @@ async function executeMaintenanceLogic(bot, context) {
2277
2361
  // Surface any over-long _gridLock hold first — observational only, and it
2278
2362
  // must run even when the rest of the tick defers on a broadcast region.
2279
2363
  checkGridLockHoldDuration(bot);
2280
- // Clear stale broadcast flag first so any downstream gating on
2281
- // isBroadcastingActive() (e.g. recalculateFunds, BTS balance check)
2282
- // sees the freshest state rather than a hung flag.
2364
+ // Placement leak watchdog first: it is the authoritative mutex and the one
2365
+ // authority that may tear a hung region down (it also drops that region's
2366
+ // stale broadcast refcount atomically). A live pipeline counter younger than
2367
+ // the bound keeps the region alive, so a slow create-group reconcile is not
2368
+ // cut out from under its own leases.
2369
+ try {
2370
+ bot.manager?._clearStalePlacementRegion?.(undefined, {
2371
+ batchInFlightMs: getPipelineInFlightAgeMs(bot),
2372
+ });
2373
+ }
2374
+ catch { /* observational */ }
2375
+ // Broadcast-flag watchdog: it refuses to act while a placement lease is
2376
+ // held, so it only clears regions that never took the exclusive region.
2283
2377
  bot.manager._clearStaleBroadcastFlag();
2378
+ checkPipelineInFlightWatchdogs(bot);
2284
2379
  await bot.manager.recalculateFunds();
2285
2380
  await checkBtsBalanceAndAcquire(bot);
2286
2381
  bot.manager.clearStalePipelineOperations();
@@ -2326,9 +2421,19 @@ async function executeMaintenanceLogic(bot, context) {
2326
2421
  // with batch broadcasts — transient mismatches are expected during a COW cycle.
2327
2422
  // Also skip while a structural recovery reload is in flight (it runs without
2328
2423
  // the fill lock, so this check must gate on the recovery flag explicitly).
2424
+ // Liveness count from the single authority; a test double that does not
2425
+ // model it falls back to the raw map size (a COUNT only — never a second
2426
+ // TTL policy), so a partial mock cannot make the gate throw.
2427
+ let pendingBroadcastCount = 0;
2428
+ try {
2429
+ pendingBroadcastCount = Number(getLivePendingBroadcasts(bot.manager)?.length) || 0;
2430
+ }
2431
+ catch {
2432
+ pendingBroadcastCount = Number(bot.manager?._pendingBroadcasts?.size) || 0;
2433
+ }
2329
2434
  if (!bot._recoverySyncInFlight &&
2330
2435
  !bot._batchInFlight &&
2331
- !(bot.manager?._pendingBroadcasts?.size > 0) &&
2436
+ pendingBroadcastCount === 0 &&
2332
2437
  (bot._lightweightSyncCheckAt == null || Date.now() - bot._lightweightSyncCheckAt >= TIMING.LIGHTWEIGHT_SYNC_CHECK_INTERVAL_MS)) {
2333
2438
  bot._lightweightSyncCheckAt = Date.now();
2334
2439
  try {
@@ -2368,6 +2473,22 @@ async function executeMaintenanceLogic(bot, context) {
2368
2473
  bot._log(`[LIGHTWEIGHT-SYNC] Check failed: ${getErrorMessage(e)}`, 'debug');
2369
2474
  }
2370
2475
  }
2476
+ // GAP-EVAC stranding sweep: a boundary commit that moved away from a live
2477
+ // order strands it inside the spread band, and plans are fill-driven — so a
2478
+ // quiet market never heals it (the band keeps live orders and the real
2479
+ // spread stays collapsed). The sweep re-derives the streak ledger against
2480
+ // the COMMITTED geometry, queues the cancel-only teeth for anything past
2481
+ // the threshold, and schedules a fill-less re-plan (the preferred repair:
2482
+ // one in-place price update). It early-returns on a healthy band, so it is
2483
+ // cheap enough to run every tick, and it also covers commits made by paths
2484
+ // that hold no bot reference (divergence/sync) plus a restart that loads an
2485
+ // already-stranded snapshot.
2486
+ try {
2487
+ scheduleGapEvacRepair(bot, `maintenance ${context}`);
2488
+ }
2489
+ catch (gapEvacErr) {
2490
+ bot._log(`[GAP-EVAC] Stranding sweep failed (non-fatal): ${getErrorMessage(gapEvacErr)}`, 'debug');
2491
+ }
2371
2492
  // Process any price corrections queued by prior sync operations before
2372
2493
  // the pipeline gate. Pending corrections block isPipelineEmpty, and no
2373
2494
  // other code path clears them outside of _consumeFillQueue (which only
@@ -2375,20 +2496,34 @@ async function executeMaintenanceLogic(bot, context) {
2375
2496
  // during startup or periodic sync can stall the pipeline indefinitely.
2376
2497
  const pendingCorrections = bot.manager.ordersNeedingPriceCorrection?.length || 0;
2377
2498
  if (pendingCorrections > 0) {
2378
- const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders);
2379
- if (correctionResult.failed > 0) {
2380
- const failedDetails = (correctionResult.results || [])
2381
- .filter((r) => !(r.result && r.result.success))
2382
- .slice(0, 3)
2383
- .map((r) => `${r.chainOrderId ?? '?'}: ${r.result?.error ?? 'unknown'}`)
2384
- .join(' | ');
2385
- bot._warn(`[MAINT] ${correctionResult.failed}/${pendingCorrections} price correction(s) failed` +
2386
- (failedDetails ? ` — ${failedDetails}` : '') +
2387
- (correctionResult.staleDropped > 0 ? ` (${correctionResult.staleDropped} stale dropped)` : ''));
2388
- }
2389
- else if (correctionResult.staleDropped > 0) {
2390
- bot._log(`[MAINT] ${correctionResult.corrected}/${pendingCorrections} price correction(s) resolved, ` +
2391
- `${correctionResult.staleDropped} stale dropped (resync moved the slot; re-queued by next sync if still off-target)`, 'info');
2499
+ // The drain cancels/updates chain orders directly, so it must own the
2500
+ // exclusive placement region. `correctAllPriceMismatches` acquires it
2501
+ // internally (single entry point); this call uses a short bound because
2502
+ // this is a background repair and must not block the maintenance tick
2503
+ // behind a long broadcast. A busy region leaves the queue for the next
2504
+ // tick (or the region-end wake-up).
2505
+ try {
2506
+ const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders, { timeoutMs: 250 });
2507
+ if (correctionResult?.deferred === true) {
2508
+ bot._log('[MAINT] Deferring price-correction drain: exclusive placement region busy or shutting down', 'info');
2509
+ }
2510
+ else if (correctionResult.failed > 0) {
2511
+ const failedDetails = (correctionResult.results || [])
2512
+ .filter((r) => !(r.result && r.result.success))
2513
+ .slice(0, 3)
2514
+ .map((r) => `${r.chainOrderId ?? '?'}: ${r.result?.error ?? 'unknown'}`)
2515
+ .join(' | ');
2516
+ bot._warn(`[MAINT] ${correctionResult.failed}/${pendingCorrections} price correction(s) failed` +
2517
+ (failedDetails ? ` — ${failedDetails}` : '') +
2518
+ (correctionResult.staleDropped > 0 ? ` (${correctionResult.staleDropped} stale dropped)` : ''));
2519
+ }
2520
+ else if (correctionResult.staleDropped > 0) {
2521
+ bot._log(`[MAINT] ${correctionResult.corrected}/${pendingCorrections} price correction(s) resolved, ` +
2522
+ `${correctionResult.staleDropped} stale dropped (resync moved the slot; re-queued by next sync if still off-target)`, 'info');
2523
+ }
2524
+ }
2525
+ catch (correctionErr) {
2526
+ bot._warn(`[MAINT] Price-correction drain failed: ${getErrorMessage(correctionErr)}`);
2392
2527
  }
2393
2528
  }
2394
2529
  // Dust detection runs before the pipeline gate. Cancellation is immediate
@@ -2764,6 +2899,14 @@ async function cancelOrderDeferredOnUncertain(bot, order) {
2764
2899
  throw err;
2765
2900
  }
2766
2901
  }
2902
+ async function runWithPlacement(bot, kind, fn, options = {}) {
2903
+ // Bot-first adapter over the ONE acquire/release contract
2904
+ // (modules/order/placement_lock.ts): it owns the bounded wait, the shutdown
2905
+ // probe, the NOOP fallback for manager-less fakes and the release. Do not
2906
+ // re-implement that here — a local copy already drifted once (it could not
2907
+ // take a lease and skipped the lease-liveness check).
2908
+ return await withPlacement(bot.manager, kind, fn, options);
2909
+ }
2767
2910
  /**
2768
2911
  * Cancel dust orders immediately — no delay, no timer, no maps.
2769
2912
  * Each dust order is cancelled on chain and its slot is rotated through
@@ -2780,52 +2923,63 @@ async function cancelDustOrders(bot, { buy: buyDust = [], sell: sellDust = [] }
2780
2923
  if (allDust.length === 0)
2781
2924
  return { cancelledCount: 0, batchResult: null };
2782
2925
  const syntheticFills = [];
2783
- for (const order of allDust) {
2784
- if (!order.orderId)
2785
- continue;
2786
- try {
2787
- const cancelResult = await cancelOrderDeferredOnUncertain(bot, order);
2926
+ // Hold the exclusive placement region across the whole cancel pass: each
2927
+ // cancel mutates the chain and the following synchronizeWithChain mutates
2928
+ // the master. A COW batch / reconcile must not interleave between them.
2929
+ // The rotation batch below runs after release (it acquires its own lease).
2930
+ const placementOk = await runWithPlacement(bot, 'dust-cancel', async () => {
2931
+ for (const order of allDust) {
2932
+ if (!order.orderId)
2933
+ continue;
2788
2934
  try {
2789
- if (cancelResult?.verifiedAfterFailure) {
2790
- const accountRef = bot.accountId || bot.account;
2791
- // The cancel was verified absent on an authoritative
2792
- // (non-empty, non-truncated) read inside cancelOrder, so an
2793
- // ambiguous refetch must not run the full sync: the snapshot
2794
- // omits the freshest orders and pass-1 phantom cleanup could
2795
- // virtualize live slots. Fall back to the local cancel sync.
2796
- const freshOrders = await readOpenOrdersGuarded(chainOrders, accountRef, {
2797
- log: (message) => bot._warn(message),
2798
- label: 'DUST',
2799
- deferEmpty: true,
2800
- skipMessage: (kind) => `[DUST] Chain refetch after verified cancel is ${kind}; applying local cancel sync for ${order.id}`,
2801
- });
2802
- if (freshOrders === null) {
2803
- await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2935
+ const cancelResult = await cancelOrderDeferredOnUncertain(bot, order);
2936
+ try {
2937
+ if (cancelResult?.verifiedAfterFailure) {
2938
+ const accountRef = bot.accountId || bot.account;
2939
+ // The cancel was verified absent on an authoritative
2940
+ // (non-empty, non-truncated) read inside cancelOrder, so an
2941
+ // ambiguous refetch must not run the full sync: the snapshot
2942
+ // omits the freshest orders and pass-1 phantom cleanup could
2943
+ // virtualize live slots. Fall back to the local cancel sync.
2944
+ const freshOrders = await readOpenOrdersGuarded(chainOrders, accountRef, {
2945
+ log: (message) => bot._warn(message),
2946
+ label: 'DUST',
2947
+ deferEmpty: true,
2948
+ skipMessage: (kind) => `[DUST] Chain refetch after verified cancel is ${kind}; applying local cancel sync for ${order.id}`,
2949
+ });
2950
+ if (freshOrders === null) {
2951
+ await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2952
+ }
2953
+ else {
2954
+ await bot.manager.synchronizeWithChain(freshOrders, 'readOpenOrders');
2955
+ }
2804
2956
  }
2805
2957
  else {
2806
- await bot.manager.synchronizeWithChain(freshOrders, 'readOpenOrders');
2958
+ await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2807
2959
  }
2808
2960
  }
2809
- else {
2810
- await bot.manager.synchronizeWithChain({ orderId: order.orderId, clearSize: true }, 'cancelOrder');
2961
+ catch (refetchErr) {
2962
+ bot._warn(`[DUST] Cancel succeeded but refetch failed for ${order.id} (${order.orderId}): ${getErrorMessage(refetchErr)}`);
2811
2963
  }
2812
- }
2813
- catch (refetchErr) {
2814
- bot._warn(`[DUST] Cancel succeeded but refetch failed for ${order.id} (${order.orderId}): ${getErrorMessage(refetchErr)}`);
2815
- }
2816
- syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
2817
- bot._log(`[DUST] Cancelled ${order.id} (${order.orderId}) size=${order.size}`, 'debug');
2818
- }
2819
- catch (err) {
2820
- const errMsg = getErrorMessage(err) || '';
2821
- if (isOrderDoesNotExistError(errMsg, order.orderId)) {
2822
2964
  syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
2823
- bot._log(`[DUST] Order ${order.id} (${order.orderId}) already gone from chain`, 'debug');
2965
+ bot._log(`[DUST] Cancelled ${order.id} (${order.orderId}) size=${order.size}`, 'debug');
2824
2966
  }
2825
- else {
2826
- bot._warn(`[DUST] Failed to cancel ${order.id} (${order.orderId}): ${errMsg}`);
2967
+ catch (err) {
2968
+ const errMsg = getErrorMessage(err) || '';
2969
+ if (isOrderDoesNotExistError(errMsg, order.orderId)) {
2970
+ syntheticFills.push({ ...order, isPartial: true, isDelayedRotationTrigger: true });
2971
+ bot._log(`[DUST] Order ${order.id} (${order.orderId}) already gone from chain`, 'debug');
2972
+ }
2973
+ else {
2974
+ bot._warn(`[DUST] Failed to cancel ${order.id} (${order.orderId}): ${errMsg}`);
2975
+ }
2827
2976
  }
2828
2977
  }
2978
+ return true;
2979
+ }, { timeoutMs: 10000, isShuttingDown: () => bot._shuttingDown === true });
2980
+ if (placementOk !== true) {
2981
+ bot._log('[DUST] Deferring dust cancels: exclusive placement region busy or shutting down', 'info');
2982
+ return { cancelledCount: 0, batchResult: null };
2829
2983
  }
2830
2984
  if (syntheticFills.length === 0)
2831
2985
  return { cancelledCount: 0, batchResult: null };
@@ -2991,19 +3145,22 @@ async function acquireBts(bot, deficit) {
2991
3145
  const sellInt = floatToBlockchainInt(best.sellAmount, best.asset.precision);
2992
3146
  const minReceiveInt = floatToBlockchainInt(minReceive, BTS_PRECISION);
2993
3147
  const op = chainOrders.buildLiquidityPoolExchangeOp(bot.accountId, best.poolId, sellInt, best.asset.id, minReceiveInt, coreAssetId);
2994
- try {
2995
- if (bot.privateKey) {
3148
+ const swapOk = await runWithPlacement(bot, 'bts-acquisition-swap', async () => {
3149
+ try {
3150
+ if (!bot.privateKey) {
3151
+ bot._log('[BTS-ACQ] CRITICAL: No signing method available', 'error');
3152
+ return false;
3153
+ }
2996
3154
  await chainOrders.executeBatch(bot.account, bot.privateKey, [op]);
3155
+ return true;
2997
3156
  }
2998
- else {
2999
- bot._log('[BTS-ACQ] CRITICAL: No signing method available', 'error');
3000
- return;
3157
+ catch (err) {
3158
+ bot._log(`[BTS-ACQ] Swap broadcast failed: ${getErrorMessage(err)}`, 'error');
3159
+ return false;
3001
3160
  }
3002
- }
3003
- catch (err) {
3004
- bot._log(`[BTS-ACQ] Swap broadcast failed: ${getErrorMessage(err)}`, 'error');
3161
+ }, { timeoutMs: 10000, isShuttingDown: () => bot._shuttingDown === true });
3162
+ if (swapOk !== true)
3005
3163
  return;
3006
- }
3007
3164
  const orderType = (best.asset.id === bot.assets?.assetA?.id) ? 'sell' : 'buy';
3008
3165
  if (bot.manager.accountant) {
3009
3166
  await bot.manager.accountant.adjustTotalBalance(orderType, -best.sellAmount, 'bts-acquisition-swap-sell');
@@ -3128,14 +3285,12 @@ async function requestGridReset(bot, reason = 'structural change', options = {})
3128
3285
  // entirely (no concurrent mutation race) instead of racing them behind a
3129
3286
  // lock.
3130
3287
  if (options.skipFillLock === true || !bot.manager._fillProcessingLock || bot.manager._fillProcessingLock.isReentrant()) {
3131
- bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
3288
+ raiseRecoveryInFlight(bot);
3132
3289
  try {
3133
3290
  return await performGridResync(bot, resetOptions);
3134
3291
  }
3135
3292
  finally {
3136
- bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
3137
- if ((bot._recoverySyncInFlight || 0) === 0)
3138
- drainFillQueueAfterPipelineClear(bot);
3293
+ lowerRecoveryInFlight(bot);
3139
3294
  }
3140
3295
  }
3141
3296
  return bot.manager._fillProcessingLock.acquire(async () => performGridResync(bot, resetOptions));
@@ -3214,13 +3369,13 @@ function wireStructuralGridResyncRequest(bot) {
3214
3369
  // prevents a fill arriving mid-reload from starting a new COW
3215
3370
  // batch whose broadcast would overlap the reload and force an
3216
3371
  // avoidable commit refusal + re-adoption cycle.
3217
- bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
3372
+ raiseRecoveryInFlight(bot);
3218
3373
  let persistedResult;
3219
3374
  try {
3220
3375
  persistedResult = await bot._recoverFromPersistedGrid();
3221
3376
  }
3222
3377
  finally {
3223
- bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
3378
+ lowerRecoveryInFlight(bot);
3224
3379
  }
3225
3380
  if (persistedResult.success) {
3226
3381
  if (bot.manager?._recoveryState) {
@@ -3245,7 +3400,7 @@ function wireStructuralGridResyncRequest(bot) {
3245
3400
  // generation orphans). _recoverFromPersistedGrid already
3246
3401
  // incremented and released the counter; increment again
3247
3402
  // for the full-resync branch.
3248
- bot._recoverySyncInFlight = (bot._recoverySyncInFlight || 0) + 1;
3403
+ raiseRecoveryInFlight(bot);
3249
3404
  let resetResult;
3250
3405
  try {
3251
3406
  resetResult = await (bot.requestGridReset('rms_structural_grid_resync', {
@@ -3258,9 +3413,7 @@ function wireStructuralGridResyncRequest(bot) {
3258
3413
  }));
3259
3414
  }
3260
3415
  finally {
3261
- bot._recoverySyncInFlight = Math.max(0, (bot._recoverySyncInFlight || 0) - 1);
3262
- if ((bot._recoverySyncInFlight || 0) === 0)
3263
- drainFillQueueAfterPipelineClear(bot);
3416
+ lowerRecoveryInFlight(bot);
3264
3417
  }
3265
3418
  if (resetResult && bot.manager?._recoveryState) {
3266
3419
  bot.manager._recoveryState = { ...bot.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
@@ -579,27 +579,39 @@ async function finishStartupSequence(bot, startupState) {
579
579
  if (shouldRegenerate) {
580
580
  if (!chainReadTruncated && Array.isArray(chainOpenOrders) && chainOpenOrders.length > 0) {
581
581
  bot._log('Generating new grid and syncing with existing on-chain orders...');
582
- await botGridModule(bot).initializeGrid(bot.manager);
583
- await bot.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
584
- // Drain queued fills BEFORE re-placement (P3 ordering): fills
585
- // replayed from history reference the PRE-restart order ids.
586
- // Processing them now (freshly synced grid) keeps their slot
587
- // provenance; processing them after reconcile re-placement
588
- // finds no grid order and falls back to orphan proceeds
589
- // credits, which drift the fund bookkeeping.
590
- if (bot._incomingFillQueue.length > 0) {
591
- bot._log(`[STARTUP] Processing ${bot._incomingFillQueue.length} queued fill(s) before re-placement (order provenance intact)`, 'info');
592
- await bot._processFillsWithBootstrapMode(chainOrders);
582
+ // Rebuild owns placement until reconcile finishes: a fill-driven
583
+ // COW that starts inside this window must defer instead of
584
+ // re-planning against the order-id-less master.
585
+ bot.manager.beginStructuralRebuild?.('startup regenerate');
586
+ try {
587
+ await botGridModule(bot).initializeGrid(bot.manager);
588
+ await bot.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
589
+ // Drain queued fills BEFORE re-placement (P3 ordering): fills
590
+ // replayed from history reference the PRE-restart order ids.
591
+ // Processing them now (freshly synced grid) keeps their slot
592
+ // provenance; processing them after reconcile re-placement
593
+ // finds no grid order and falls back to orphan proceeds
594
+ // credits, which drift the fund bookkeeping.
595
+ if (bot._incomingFillQueue.length > 0) {
596
+ bot._log(`[STARTUP] Processing ${bot._incomingFillQueue.length} queued fill(s) before re-placement (order provenance intact)`, 'info');
597
+ await bot._processFillsWithBootstrapMode(chainOrders);
598
+ }
599
+ const rebalanceResult = await reconcileMod.reconcileGridOrders({
600
+ manager: bot.manager,
601
+ config: bot.config,
602
+ account: bot.account,
603
+ privateKey: bot.privateKey,
604
+ chainOrders,
605
+ chainOpenOrders,
606
+ // This path IS the rebuild: it owns update-first
607
+ // placement and must not defer on its own flag.
608
+ options: { allowDuringRebuild: true },
609
+ });
610
+ await bot._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
611
+ }
612
+ finally {
613
+ bot.manager.endStructuralRebuild?.();
593
614
  }
594
- const rebalanceResult = await reconcileMod.reconcileGridOrders({
595
- manager: bot.manager,
596
- config: bot.config,
597
- account: bot.account,
598
- privateKey: bot.privateKey,
599
- chainOrders,
600
- chainOpenOrders,
601
- });
602
- await bot._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
603
615
  }
604
616
  else if (!chainReadTruncated) {
605
617
  bot._log('Generating new grid and placing initial orders on-chain...');
@@ -98,6 +98,38 @@ declare function recoverFromPersistedGrid(bot: BotLike): Promise<{
98
98
  reason: string;
99
99
  halt: boolean;
100
100
  }>;
101
+ /**
102
+ * Detect live orders resting INSIDE the committed gap band and schedule the
103
+ * repair (fill-less COW re-plan + cancel-only teeth as last resort).
104
+ *
105
+ * Why this exists: a boundary commit that moves the boundary away from a live
106
+ * order strands it inside the spread band. The plan normally evacuates it with
107
+ * a rotation in the SAME batch, but the LAST-FILL-GUARD re-proves an evacuation
108
+ * against the PRE-COMMIT geometry and refuses it as stale when the plan's own
109
+ * crawl is what created the stranding (fail-closed by design — see the
110
+ * 2026-10-30 bid-edge incident). The batch then commits the new boundary with
111
+ * the stranded order still live, and because plans are fill-driven the state
112
+ * never heals while the market is quiet: the 4-slot spread band keeps two live
113
+ * buys and the real spread collapses (observed: 1.21% against a 2.0% target).
114
+ *
115
+ * Re-planning AFTER the commit is safe where the in-batch rotation is not: the
116
+ * committed geometry is now the LIVE geometry, so the evacuation stamp passes
117
+ * `isEvacuationStampStillValid` and the rotation executes as one price update.
118
+ *
119
+ * Cheap when healthy: one `detectGapEvacuationCandidates` scan (O(grid)) and an
120
+ * early return. Callers run it on commit (writer of the stranding) and on the
121
+ * maintenance tick (covers restarts and commits from paths without a bot
122
+ * reference).
123
+ *
124
+ * @param {import('./dexbot_class.js').DEXBot} bot
125
+ * @param {string} context - Label for logging/telemetry
126
+ * @returns `{ stranded, scheduled }` — stranded slot ids found, whether a
127
+ * re-plan was scheduled.
128
+ */
129
+ declare function scheduleGapEvacRepair(bot: BotLike, context?: string): {
130
+ stranded: string[];
131
+ scheduled: boolean;
132
+ };
101
133
  /**
102
134
  * Schedule a best-effort target-grid rebalance after a recovery sync.
103
135
  *
@@ -126,5 +158,5 @@ declare function rejectCorruptedGridSnapshot(bot: BotLike, context: string): Pro
126
158
  * @returns {Promise<boolean>}
127
159
  */
128
160
  declare function targetedOrderRepair(bot: BotLike, orderIds: string[]): Promise<boolean>;
129
- export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
161
+ export { persistAndRecoverIfNeeded, getRecentFillKeysSnapshot, triggerStateRecoverySync, abortFlowIfIllegalState, handleBatchHardAbort, applyRecoverableGridUpdates, recoverExplicitStaleOrders, recoverBatchSizeDrift, recoverFromPersistedGrid, rejectCorruptedGridSnapshot, targetedOrderRepair, scheduleGapEvacRepair, _schedulePostRecoveryRebalance as schedulePostRecoveryRebalance };
130
162
  //# sourceMappingURL=dexbot_state_recovery.d.ts.map