dexbot 1.7.2 → 1.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/claw/package.json +1 -1
  3. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  4. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  5. package/dist/analysis/grid_correction_check.d.ts +60 -1
  6. package/dist/analysis/grid_correction_check.js +268 -18
  7. package/dist/analysis/pivot_timeline.d.ts +78 -0
  8. package/dist/analysis/pivot_timeline.js +360 -0
  9. package/dist/market_adapter/utils/file_lock.js +16 -4
  10. package/dist/modules/account_bots.d.ts +6 -0
  11. package/dist/modules/bot_defaults.js +6 -1
  12. package/dist/modules/constants.d.ts +12 -0
  13. package/dist/modules/constants.js +66 -0
  14. package/dist/modules/dexbot_class.d.ts +17 -0
  15. package/dist/modules/dexbot_class.js +56 -1
  16. package/dist/modules/dexbot_cow_runtime.js +231 -19
  17. package/dist/modules/dexbot_maintenance_runtime.d.ts +31 -1
  18. package/dist/modules/dexbot_maintenance_runtime.js +164 -11
  19. package/dist/modules/dexbot_state_recovery.js +15 -1
  20. package/dist/modules/launcher/instance_lock.d.ts +27 -0
  21. package/dist/modules/launcher/instance_lock.js +131 -0
  22. package/dist/modules/order/grid.d.ts +19 -0
  23. package/dist/modules/order/grid.js +40 -14
  24. package/dist/modules/order/index.d.ts +15 -1
  25. package/dist/modules/order/manager.d.ts +21 -0
  26. package/dist/modules/order/manager.js +119 -12
  27. package/dist/modules/order/placement_lock.d.ts +30 -0
  28. package/dist/modules/order/placement_lock.js +48 -0
  29. package/dist/modules/order/utils/order.d.ts +39 -1
  30. package/dist/modules/order/utils/order.js +73 -2
  31. package/dist/modules/order/utils/system.d.ts +112 -3
  32. package/dist/modules/order/utils/system.js +368 -6
  33. package/dist/modules/paths.d.ts +1 -0
  34. package/dist/modules/paths.js +1 -0
  35. package/dist/modules/timing_policy.js +2 -0
  36. package/dist/modules/types.d.ts +27 -0
  37. package/dist/pm2.js +126 -0
  38. package/dist/unlock.js +26 -1
  39. package/docs/DEXBOT_COMPARISON.md +5 -5
  40. package/docs/EVOLUTION.md +10 -9
  41. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  42. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +155 -2
  43. package/docs/README.md +1 -1
  44. package/modules/README.md +1 -0
  45. package/package.json +2 -1
@@ -8,7 +8,7 @@ const require = createRequire(import.meta.url);
8
8
  import { lazyShim } from './lazy_shim.js';
9
9
  import fs from 'node:fs';
10
10
  import { spawn } from 'node:child_process';
11
- import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills, resolveGridUpdateBlockers, noteGridUpdateBlockers } from './order/utils/system.js';
11
+ import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills, resolveGridUpdateBlockers, noteGridUpdateBlockers, expireLastFillGuardReferences } from './order/utils/system.js';
12
12
  import { withPlacement } from './order/placement_lock.js';
13
13
  import { path } from './path_api.js';
14
14
  import * as chainOrders from './chain_orders.js';
@@ -2662,6 +2662,7 @@ async function executeMaintenanceLogic(bot, context) {
2662
2662
  }
2663
2663
  else {
2664
2664
  const spreadResult = await bot.manager.checkSpreadCondition(BitShares, bot.updateOrdersOnChainPlan.bind(bot));
2665
+ const spreadExcessAfter = Number(spreadResult?.excessAfter);
2665
2666
  if (await bot._abortFlowIfIllegalState(`${context} spread check`))
2666
2667
  return;
2667
2668
  // Mirror the stall cause and arm the targeted-sync refresh for
@@ -2678,7 +2679,9 @@ async function executeMaintenanceLogic(bot, context) {
2678
2679
  }
2679
2680
  // Persistence watchdog: a correction that keeps placing nothing
2680
2681
  // while the spread stays wide is a stale grid, not patience.
2681
- trackOutOfSpreadStaleness(bot, true, spreadPlaced, spreadResult?.transient === true);
2682
+ // `excessAfter` is the post-placement violation, so a placement
2683
+ // that did not reduce it no longer counts as progress.
2684
+ trackOutOfSpreadStaleness(bot, true, spreadPlaced, spreadResult?.transient === true, (spreadResult?.excessAfter !== undefined && Number.isFinite(spreadExcessAfter)) ? spreadExcessAfter : undefined);
2682
2685
  }
2683
2686
  }
2684
2687
  catch (err) {
@@ -2706,6 +2709,75 @@ function clearGeometryResyncMemory(bot) {
2706
2709
  bot._lastGeometryResyncPivotKey = '';
2707
2710
  bot._geometryResyncWarnedPivotKey = '';
2708
2711
  }
2712
+ /**
2713
+ * Arm a one-shot, level-triggered spread recheck (TIMING.SPREAD_STALE_RECHECK_MS).
2714
+ *
2715
+ * Retrospective R5.5: "Make deferred work level-triggered — never let a retry
2716
+ * depend on a future event." The spread check has no driver of its own: it runs
2717
+ * post-fill and on the blockchain-fetch tick (240 min by default, and the
2718
+ * open-orders loop that would shorten it is OFF by default). Every watchdog
2719
+ * window (warn 10 min, geometry 10 min, fund 5 min, pivot-expiry 20 min) is
2720
+ * therefore unreachable while the grid sits out of tolerance, which is how a
2721
+ * live bot went 86 minutes without a single check. While out of tolerance the
2722
+ * watchdog now schedules its own retry, so the clocks run at any base cadence.
2723
+ *
2724
+ * Deduped and shutdown-safe: one timer at a time, never while a recheck is in
2725
+ * flight, never while shutting down. `performPeriodicGridChecks` keeps its own
2726
+ * pipeline/broadcast deferrals, so this cannot interleave with a fill cycle.
2727
+ *
2728
+ * @param {Object} bot - DEXBot instance
2729
+ * @param {number} [delayOverrideMs] - Injectable delay (tests); production reads TIMING
2730
+ * @returns {boolean} True when a recheck was armed (or already pending)
2731
+ */
2732
+ export function scheduleSpreadRecheck(bot, delayOverrideMs) {
2733
+ const configured = Number.isFinite(Number(delayOverrideMs)) && Number(delayOverrideMs) > 0
2734
+ ? Number(delayOverrideMs)
2735
+ : (Number(TIMING?.SPREAD_STALE_RECHECK_MS) > 0 ? Number(TIMING.SPREAD_STALE_RECHECK_MS) : 0);
2736
+ const ms = configured > 0 ? configured : 0;
2737
+ if (!(ms > 0))
2738
+ return false;
2739
+ if (!bot || bot._shuttingDown)
2740
+ return false;
2741
+ if (bot._spreadRecheckTimer || bot._spreadRecheckInFlight)
2742
+ return true;
2743
+ try {
2744
+ bot._spreadRecheckTimer = setTimeout(async () => {
2745
+ bot._spreadRecheckTimer = null;
2746
+ if (bot._shuttingDown || bot._spreadRecheckInFlight)
2747
+ return;
2748
+ bot._spreadRecheckInFlight = true;
2749
+ try {
2750
+ bot._log(`[SPREAD-STALE] Level-triggered recheck (${Math.round(ms / 1000)}s cadence while out of tolerance)`, 'debug');
2751
+ await performPeriodicGridChecks(bot);
2752
+ }
2753
+ catch (err) {
2754
+ try {
2755
+ bot._warn(`Spread recheck failed: ${getErrorMessage(err)}`);
2756
+ }
2757
+ catch { /* logging is best-effort */ }
2758
+ }
2759
+ finally {
2760
+ bot._spreadRecheckInFlight = false;
2761
+ // `trackOutOfSpreadStaleness` runs inside the maintenance call,
2762
+ // while this flag is true, so its dedupe check intentionally
2763
+ // does not arm the next timer. Re-arm after the region ends when
2764
+ // the level is still active; otherwise a single retry would be
2765
+ // falsely documented as a recurring level-triggered repair.
2766
+ if (!bot._shuttingDown && Number(bot.manager?.outOfSpread) > 0) {
2767
+ scheduleSpreadRecheck(bot, ms);
2768
+ }
2769
+ }
2770
+ }, ms);
2771
+ const timer = bot._spreadRecheckTimer;
2772
+ if (typeof timer?.unref === 'function')
2773
+ timer.unref();
2774
+ return true;
2775
+ }
2776
+ catch {
2777
+ bot._spreadRecheckTimer = null;
2778
+ return false;
2779
+ }
2780
+ }
2709
2781
  /**
2710
2782
  * Out-of-spread persistence watchdog: the never-run-stale backstop.
2711
2783
  *
@@ -2724,19 +2796,44 @@ function clearGeometryResyncMemory(bot) {
2724
2796
  * @param {number} ordersPlaced - Correction orders placed this tick
2725
2797
  * @param {boolean} [transient=false] - True when the correction was refused for
2726
2798
  * a retryable reason (stale plan / duplicate CREATE / unclassified non-execution)
2799
+ * @param {number} [excessAfter] - Out-of-spread violation (grid steps) MEASURED
2800
+ * after the placement, or undefined when the correction placed nothing /
2801
+ * could not be measured
2727
2802
  * @returns {{staleMs: number, escalated: boolean}}
2728
2803
  */
2729
- export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, transient = false) {
2804
+ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, transient = false, excessAfter) {
2730
2805
  const outOfSpread = Number(bot?.manager?.outOfSpread) || 0;
2731
2806
  const nowMs = Date.now();
2732
2807
  const inToleranceResetMs = Number(TIMING?.SPREAD_STALE_IN_TOLERANCE_RESET_MS) > 0
2733
2808
  ? Number(TIMING.SPREAD_STALE_IN_TOLERANCE_RESET_MS)
2734
2809
  : 10 * 60 * 1000;
2735
- if (ordersPlaced > 0) {
2810
+ // PLACED IS NOT PROGRESS (retrospective §9.11.3, "a repair must reduce the
2811
+ // violation"). Clearing the clock on ANY placement let a correction that
2812
+ // only rotated the side that was not blocking report a heal every cycle:
2813
+ // live, 6 buy rotations at a still-3.0% spread reset the clock AND the
2814
+ // geometry-recenter memory, so the pivot expiry and the one-recenter-per-pivot
2815
+ // budget could never fire while the sell side stayed unquoted for 86 minutes.
2816
+ // A placement now clears the clock only when the measured violation actually
2817
+ // shrank (excessAfter < outOfSpread, or 0). Unmeasurable => legacy behavior
2818
+ // (a placement counts), so a broken measurement can never churn a re-center.
2819
+ const measured = Number(excessAfter);
2820
+ const excessKnown = excessAfter !== undefined && excessAfter !== null && Number.isFinite(measured);
2821
+ const placedReducedViolation = ordersPlaced > 0 && (!excessKnown || measured < outOfSpread);
2822
+ if (ordersPlaced > 0 && !placedReducedViolation) {
2823
+ if (!bot._outOfSpreadNoProgressWarned) {
2824
+ bot._outOfSpreadNoProgressWarned = true;
2825
+ bot._log(`[SPREAD-STALE] ${ordersPlaced} order(s) placed but the spread is still ${measured} step(s) `
2826
+ + `out of tolerance (was ${outOfSpread}) — counting as no progress, the stale clock keeps running`, 'warn');
2827
+ }
2828
+ // Fall through: the clock is NOT cleared, so the documented windows
2829
+ // (warn / geometry / fund / pivot-expiry) stay reachable.
2830
+ }
2831
+ if (placedReducedViolation) {
2736
2832
  bot._outOfSpreadSince = 0;
2737
2833
  bot._outOfSpreadStaleWarned = false;
2738
2834
  bot._outOfSpreadTransientCount = 0;
2739
2835
  bot._outOfSpreadToleranceSince = 0;
2836
+ bot._outOfSpreadNoProgressWarned = false;
2740
2837
  clearGeometryResyncMemory(bot);
2741
2838
  try {
2742
2839
  bot.manager._spreadRailOversubscribed = null;
@@ -2763,6 +2860,7 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2763
2860
  bot._outOfSpreadStaleWarned = false;
2764
2861
  bot._outOfSpreadTransientCount = 0;
2765
2862
  bot._outOfSpreadToleranceSince = 0;
2863
+ bot._outOfSpreadNoProgressWarned = false;
2766
2864
  clearGeometryResyncMemory(bot);
2767
2865
  try {
2768
2866
  bot.manager._spreadRailOversubscribed = null;
@@ -2771,6 +2869,13 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2771
2869
  return { staleMs: 0, escalated: false };
2772
2870
  }
2773
2871
  bot._outOfSpreadToleranceSince = 0;
2872
+ // LEVEL-TRIGGERED RECHECK. Every watchdog clock below assumes the spread
2873
+ // check is EVALUATED, but the base cadence is the blockchain fetch interval
2874
+ // (240 min by default) plus the post-fill path — so a grid that is out of
2875
+ // tolerance waits hours for the next look and every window becomes dead
2876
+ // code (live: 86 minutes with no check at all). While out of tolerance, arm
2877
+ // one short recheck; it re-arms itself until the spread heals.
2878
+ scheduleSpreadRecheck(bot);
2774
2879
  if (!spreadChecked) {
2775
2880
  const since = Number(bot._outOfSpreadSince) || 0;
2776
2881
  return { staleMs: since > 0 ? Date.now() - since : 0, escalated: false };
@@ -2840,12 +2945,27 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2840
2945
  const geometryEscalateMs = Number(TIMING?.SPREAD_STALE_GEOMETRY_ESCALATE_MS) > 0
2841
2946
  ? Number(TIMING.SPREAD_STALE_GEOMETRY_ESCALATE_MS)
2842
2947
  : 10 * 60 * 1000;
2948
+ // After the one re-center per pivot, the pivot gets a BOUNDED extra window
2949
+ // (SPREAD_STALE_PIVOT_EXPIRE_MS) instead of escalating to NOTHING ("a fill
2950
+ // or the pivot TTL is the only heal"). Live, a single off-market pivot buy
2951
+ // stranded the grid out of spread for the full 24h TTL because every
2952
+ // rebuild re-armed the same pivot. Past the window the guard references
2953
+ // are expired outright (expireLastFillGuardReferences) so spread
2954
+ // correction trades around the live market again; both re-arm on the next
2955
+ // real fill. 0 disables the expiry (legacy behavior).
2956
+ const pivotExpireConfigured = Number(TIMING?.SPREAD_STALE_PIVOT_EXPIRE_MS);
2957
+ const pivotExpireMs = pivotExpireConfigured > 0
2958
+ ? pivotExpireConfigured
2959
+ : pivotExpireConfigured === 0
2960
+ ? Number.POSITIVE_INFINITY
2961
+ : 20 * 60 * 1000;
2843
2962
  const escalateMs = oversubscribedFresh
2844
2963
  ? fundEscalateMs
2845
2964
  : geometryAlreadyRecentered
2846
- // Do not re-center again for the same fill. Out of the escalation
2847
- // ladder entirely: a fill (or the pivot TTL) is the only heal.
2848
- ? Number.POSITIVE_INFINITY
2965
+ // Do not re-center again for the same fill. Bounded pivot-expiry
2966
+ // window instead of Infinity: past it the guard references are
2967
+ // cleared and the market is traded around (see pivotExpireMs).
2968
+ ? pivotExpireMs
2849
2969
  : geometryBlocked
2850
2970
  ? geometryEscalateMs
2851
2971
  : (Number(TIMING?.SPREAD_STALE_ESCALATE_MS) > 0
@@ -2877,6 +2997,19 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2877
2997
  const lastAt = Number(bot._lastSpreadStaleResyncAt) || 0;
2878
2998
  if (now - lastAt >= cooldownMs) {
2879
2999
  bot._lastSpreadStaleResyncAt = now;
3000
+ if (geometryAlreadyRecentered) {
3001
+ // The re-center for this pivot already ran and cannot clear the
3002
+ // veto; the bounded pivot-expiry window is up. Clear the guard
3003
+ // references (live + persisted) so the market is traded around
3004
+ // again instead of staying stranded until the 24h TTL.
3005
+ expireLastFillGuardReferences(bot.manager, `out of tolerance ${staleMin}min after re-centering on fill pivot ${geometryPivotKey.split('|')[0]}`);
3006
+ clearGeometryResyncMemory(bot);
3007
+ bot._outOfSpreadSince = 0;
3008
+ bot._outOfSpreadStaleWarned = false;
3009
+ bot._outOfSpreadTransientCount = 0;
3010
+ bot._outOfSpreadToleranceSince = 0;
3011
+ return { staleMs, escalated: true };
3012
+ }
2880
3013
  if (geometryBlocked && geometryPivotKey)
2881
3014
  bot._lastGeometryResyncPivotKey = geometryPivotKey;
2882
3015
  bot._log(`[SPREAD-STALE] Spread out of tolerance for ${staleMin}min despite corrections` +
@@ -3356,7 +3489,26 @@ function wireStructuralGridResyncRequest(bot) {
3356
3489
  // commit refusal + re-adoption cycle. Defer until the batch
3357
3490
  // completes; the fill consumer is already gated on _batchInFlight,
3358
3491
  // so this only blocks the recovery itself, never new fills.
3359
- if (bot._batchInFlight > 0) {
3492
+ //
3493
+ // SAME DEFER FOR AN UNSETTLED FILL PIPELINE: a fill batch mutates
3494
+ // the grid in memory long before its persistence flush reaches
3495
+ // disk, so the on-disk snapshot can still describe the PRE-fill
3496
+ // slot sizes. Reloading it rolls an applied fill back, and the next
3497
+ // fill for the same order is then subtracted from that rolled-back
3498
+ // baseline — leaving a phantom residual plus an orderId the chain
3499
+ // no longer has. Observed live: two partial fills of one order in
3500
+ // consecutive blocks; the reload between them turned a completed
3501
+ // order into a phantom parked in the gap band, which froze the
3502
+ // boundary crawl so the spread could not be closed.
3503
+ //
3504
+ // `hasUnsettledFills` is the same predicate the destructive
3505
+ // snapshot-reject gate uses: only a settled fill pipeline makes the
3506
+ // on-disk snapshot a safe base.
3507
+ const fillPipeline = hasUnsettledFills(bot);
3508
+ const deferCause = bot._batchInFlight > 0
3509
+ ? `_batchInFlight=${bot._batchInFlight}`
3510
+ : (fillPipeline.unsettled ? `fill pipeline not settled: ${fillPipeline.reason}` : null);
3511
+ if (deferCause) {
3360
3512
  const now = Date.now();
3361
3513
  if (!bot._structuralGridResyncDeferStartedAt) {
3362
3514
  bot._structuralGridResyncDeferStartedAt = now;
@@ -3365,7 +3517,7 @@ function wireStructuralGridResyncRequest(bot) {
3365
3517
  bot._structuralGridResyncDeferCount = (bot._structuralGridResyncDeferCount || 0) + 1;
3366
3518
  const deferredMs = now - bot._structuralGridResyncDeferStartedAt;
3367
3519
  if (deferredMs >= STRUCTURAL_RESYNC_MAX_DEFER_MS) {
3368
- bot._warn(`[RECOVERY] Structural resync max-defer reached (${deferredMs}ms, cap ${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms, deferred ${bot._structuralGridResyncDeferCount}x) — forcing despite _batchInFlight=${bot._batchInFlight}: ${reason}`);
3520
+ bot._warn(`[RECOVERY] Structural resync max-defer reached (${deferredMs}ms, cap ${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms, deferred ${bot._structuralGridResyncDeferCount}x) — forcing despite ${deferCause}: ${reason}`);
3369
3521
  bot._structuralGridResyncDeferStartedAt = null;
3370
3522
  bot._structuralGridResyncDeferCount = 0;
3371
3523
  }
@@ -3375,10 +3527,10 @@ function wireStructuralGridResyncRequest(bot) {
3375
3527
  // defer lines in seconds); count the rest and surface them only in
3376
3528
  // the final forced message.
3377
3529
  if (bot._structuralGridResyncDeferCount === 1) {
3378
- bot._warn(`[RECOVERY] Structural resync defer (batchInFlight=${bot._batchInFlight}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`);
3530
+ bot._warn(`[RECOVERY] Structural resync defer (${deferCause}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`);
3379
3531
  }
3380
3532
  else {
3381
- bot.manager?.logger?.log?.(`[RECOVERY] Structural resync defer (suppressed; count=${bot._structuralGridResyncDeferCount}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`, 'debug');
3533
+ bot.manager?.logger?.log?.(`[RECOVERY] Structural resync defer (suppressed; ${deferCause}, count=${bot._structuralGridResyncDeferCount}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`, 'debug');
3382
3534
  }
3383
3535
  bot._structuralGridResyncTimer = setTimeout(runStructuralResync, TIMING.LOCK_REFRESH_MIN_MS);
3384
3536
  return;
@@ -3615,6 +3767,7 @@ export default {
3615
3767
  handlePendingTriggerReset,
3616
3768
  setupTriggerFileDetection,
3617
3769
  performPeriodicGridChecks,
3770
+ scheduleSpreadRecheck,
3618
3771
  isOpenOrdersSyncLoopEnabled,
3619
3772
  startOpenOrdersSyncLoop,
3620
3773
  stopOpenOrdersSyncLoop,
@@ -11,6 +11,7 @@ import { restoreGapEvacStreaks, applyPersistedPendingCrawls, resetLastFillPivot,
11
11
  import { blockchainToFloat, calculateGapSlots, resolveGapSlots, validatePersistedBoundary, isTransientInBandRejection } from './order/utils/math.js';
12
12
  import { ON_MISSING_GENESIS, isMissingGenesisError } from './order/genesis_policy.js';
13
13
  import { hasExecutableActions } from './order/utils/validate.js';
14
+ import { releasePlanLease } from './order/placement_lock.js';
14
15
  import { getErrorMessage } from './utils/errors.js';
15
16
  const { isGridBloated } = grid;
16
17
  /**
@@ -643,7 +644,10 @@ async function _rebalanceAfterRecovery(bot, context) {
643
644
  return;
644
645
  }
645
646
  try {
646
- const rebalanceResult = await bot.manager.performSafeRebalance([], new Set());
647
+ // holdPlanRegion: every result on this path is either executed below or
648
+ // released explicitly, so the plan→commit window is serialized against
649
+ // the fill-driven planner.
650
+ const rebalanceResult = await bot.manager.performSafeRebalance([], new Set(), { holdPlanRegion: true });
647
651
  if (!rebalanceResult || rebalanceResult.aborted) {
648
652
  // A deferred abort means owed work is still outstanding (broadcast
649
653
  // held the plan, or the identical-held-plan suppression is waiting
@@ -661,9 +665,19 @@ async function _rebalanceAfterRecovery(bot, context) {
661
665
  // rebalance stack stays balanced and _rebalanceState returns to
662
666
  // NORMAL (mirrors _executeBatchIfNeeded's no-action pop). Aborted
663
667
  // results are never pushed, so only this branch needs the pop.
668
+ // The result also owns the plan region acquired before planning;
669
+ // no executor will adopt it on this path, so release it here.
670
+ releasePlanLease(rebalanceResult);
664
671
  bot.manager?._popWorkingGridRef?.(rebalanceResult);
665
672
  return;
666
673
  }
674
+ const railCount = Array.isArray(rebalanceResult.actions) ? rebalanceResult.actions.length : 0;
675
+ if (railCount > 0 && bot._metrics) {
676
+ // Cost of a recovery reload/repair in re-placed rails. Counted here
677
+ // (not derived from logs) so the reload→re-place amplification is
678
+ // observable next to the other churn counters.
679
+ bot._metrics.reloadReplacedRailCount = (bot._metrics.reloadReplacedRailCount || 0) + railCount;
680
+ }
667
681
  bot.manager.logger?.log?.(`[RECOVERY] Re-placing ${(rebalanceResult.actions ?? []).length} missing rail(s) after ${context}`, 'info');
668
682
  await bot.updateOrdersOnChainBatch(rebalanceResult);
669
683
  }
@@ -0,0 +1,27 @@
1
+ /** A held instance lock. release() is ownership-checked and idempotent. */
2
+ export interface BotInstanceLock {
3
+ release(): void;
4
+ /** Absolute path of the lock file (diagnostics/tests). */
5
+ lockPath: string;
6
+ }
7
+ export interface AcquireInstanceLockOptions {
8
+ /** Best-effort logger (manager logger or launcher console shim). */
9
+ log?: (message: string, level?: string) => void;
10
+ }
11
+ /**
12
+ * Acquire the single-controller lock for `account`.
13
+ *
14
+ * Throws (never waits) when another LIVE process holds the lock — the error
15
+ * message names the holder (pid, bot, start time) so the operator can stop the
16
+ * right instance. A DEAD holder (crashed process, recycled pid) is detected
17
+ * via /proc liveness + starttime match and its stale lock is taken over
18
+ * immediately, so a PM2 restart after a crash is not blocked by its own
19
+ * previous incarnation.
20
+ *
21
+ * @param {string} account - Chain account name (lock key)
22
+ * @param {string} botName - Configured bot name (diagnostics only)
23
+ * @param {Object} [options]
24
+ * @returns {BotInstanceLock} Handle with release()
25
+ */
26
+ export declare function acquireBotInstanceLock(account: string, botName?: string, options?: AcquireInstanceLockOptions): BotInstanceLock;
27
+ //# sourceMappingURL=instance_lock.d.ts.map
@@ -0,0 +1,131 @@
1
+ 'use strict';
2
+ // node-only entry point — per-account bot instance lock (/proc liveness, fs)
3
+ /**
4
+ * instance_lock.ts - Single-controller lock per trading account
5
+ *
6
+ * Prevents two bot processes (e.g. one started via `dexbot pm2` and one via a
7
+ * normal `dexbot start` unlock) from controlling the SAME account at the same
8
+ * time. Observed live (2026-11-14): two controllers kept independent in-memory
9
+ * grids over the same 40 chain orders, cancelled each other's orders as
10
+ * "duplicate orphans", attributed the same fill to different slots, and
11
+ * finally stranded the grid out of spread for ~24h behind an off-market
12
+ * LAST-FILL-GUARD pivot. This lock makes the second controller fail fast with
13
+ * the holder's identity instead of silently double-trading.
14
+ *
15
+ * The lock is keyed by ACCOUNT, not bot name: the conflict domain is the chain
16
+ * account's open orders and fills, so even two differently named bot entries
17
+ * sharing an account are refused.
18
+ *
19
+ * Implementation reuses the market adapter's file-lock primitive (staleness +
20
+ * heartbeat + ownership-checked release) with a process-liveness aliveCheck
21
+ * that also compares /proc/<pid>/stat starttime, so a recycled pid can neither
22
+ * keep a dead holder's lock alive nor steal a live holder's lock.
23
+ */
24
+ import { path } from '../path_api.js';
25
+ import { getStorage } from '../storage/index.js';
26
+ import { PATHS } from '../paths.js';
27
+ import { runtime } from '../runtime.js';
28
+ import { getProcessDiscovery } from '../process_discovery.js';
29
+ import { acquireFileLockSync, releaseFileLockSync } from '../../market_adapter/utils/file_lock.js';
30
+ import { getErrorMessage } from '../utils/errors.js';
31
+ const storage = getStorage();
32
+ function sanitizeLockKey(raw) {
33
+ return String(raw || 'unknown').toLowerCase().replace(/[^a-z0-9._-]+/g, '_').slice(0, 64) || 'unknown';
34
+ }
35
+ function instanceLocksDir() {
36
+ return PATHS.PROFILES.INSTANCE_LOCKS_DIR || path.join(PATHS.PROFILES_DIR, 'instance-locks');
37
+ }
38
+ function lockPathForAccount(account) {
39
+ return path.join(instanceLocksDir(), `${sanitizeLockKey(account)}.lock`);
40
+ }
41
+ function readLockPayload(lockPath) {
42
+ try {
43
+ const parsed = storage.readJSON(lockPath);
44
+ return parsed && typeof parsed === 'object' ? parsed : null;
45
+ }
46
+ catch (_) {
47
+ return null;
48
+ }
49
+ }
50
+ /**
51
+ * Acquire the single-controller lock for `account`.
52
+ *
53
+ * Throws (never waits) when another LIVE process holds the lock — the error
54
+ * message names the holder (pid, bot, start time) so the operator can stop the
55
+ * right instance. A DEAD holder (crashed process, recycled pid) is detected
56
+ * via /proc liveness + starttime match and its stale lock is taken over
57
+ * immediately, so a PM2 restart after a crash is not blocked by its own
58
+ * previous incarnation.
59
+ *
60
+ * @param {string} account - Chain account name (lock key)
61
+ * @param {string} botName - Configured bot name (diagnostics only)
62
+ * @param {Object} [options]
63
+ * @returns {BotInstanceLock} Handle with release()
64
+ */
65
+ export function acquireBotInstanceLock(account, botName = '', options = {}) {
66
+ const lockPath = lockPathForAccount(account);
67
+ const discovery = getProcessDiscovery();
68
+ try {
69
+ storage.ensureDir(instanceLocksDir());
70
+ }
71
+ catch (err) {
72
+ throw new Error(`Cannot create instance-lock directory ${instanceLocksDir()}: ${getErrorMessage(err)}`);
73
+ }
74
+ const buildPayload = () => ({
75
+ pid: runtime.pid,
76
+ botName: String(botName || ''),
77
+ account: String(account || ''),
78
+ starttime: discovery.readStat(runtime.pid)?.starttime ?? 0,
79
+ at: new Date().toISOString(),
80
+ });
81
+ const aliveCheck = (pid) => {
82
+ if (!discovery.isAlive(pid))
83
+ return false;
84
+ // pid-recycle guard: the payload's starttime is the holder's boot
85
+ // identity. A recycled pid whose /proc starttime differs is NOT the
86
+ // holder — treat the lock as dead so a new controller can take over.
87
+ const info = readLockPayload(lockPath);
88
+ const holderStart = Number(info?.starttime);
89
+ if (Number(info?.pid) === pid && holderStart > 0) {
90
+ const stat = discovery.readStat(pid);
91
+ if (stat && stat.starttime !== holderStart)
92
+ return false;
93
+ }
94
+ return true;
95
+ };
96
+ const alreadyRunningMsg = () => {
97
+ const info = readLockPayload(lockPath);
98
+ const holderBot = info?.botName ? ` (bot ${info.botName})` : '';
99
+ const holderAt = info?.at ? `, started ${info.at}` : '';
100
+ return (`account '${account}' is already controlled by another bot process ` +
101
+ `(pid ${info?.pid ?? 'unknown'}${holderBot}${holderAt}, lock: ${lockPath}). ` +
102
+ `Stop that instance first (dexbot pm2 stop / dexbot stop) — two controllers ` +
103
+ `on one account corrupt each other's grids and fills.`);
104
+ };
105
+ const lock = acquireFileLockSync(lockPath, {
106
+ // Fail fast — the second controller must not wait out a live holder.
107
+ contention: 'throw',
108
+ // Only bounds an ALIVE-but-never-beating holder (heartbeat stopped
109
+ // without process death); a dead holder is stolen immediately by the
110
+ // aliveCheck above regardless of staleness.
111
+ staleMs: 10 * 60 * 1000,
112
+ aliveCheck,
113
+ buildPayload,
114
+ alreadyRunningMsg,
115
+ timeoutMsg: `cannot acquire instance lock for account '${account}': ${lockPath}`,
116
+ });
117
+ options.log?.(`Instance lock acquired for account '${account}' (bot ${botName || 'n/a'}, pid ${runtime.pid})`, 'info');
118
+ return {
119
+ lockPath,
120
+ release() {
121
+ try {
122
+ releaseFileLockSync({ fd: lock.fd, lockPath: lock.lockPath, heartbeat: lock.heartbeat, ownerId: lock.ownerId });
123
+ options.log?.(`Instance lock released for account '${account}'`, 'info');
124
+ }
125
+ catch (err) {
126
+ options.log?.(`Instance lock release failed for account '${account}': ${getErrorMessage(err)}`, 'warn');
127
+ }
128
+ },
129
+ };
130
+ }
131
+ //# sourceMappingURL=instance_lock.js.map
@@ -654,6 +654,24 @@ export declare function monitorDivergence(manager: OrderManagerLike, calculatedG
654
654
  * @returns {number} The calculated spread percentage.
655
655
  */
656
656
  export declare function calculateCurrentSpread(manager: OrderManagerLike): number;
657
+ /**
658
+ * Measure the current out-of-spread violation in grid steps, from the live
659
+ * on-chain window only. ONE implementation for the entry reading and the
660
+ * post-placement reading, so "did the repair reduce the violation?" is the
661
+ * same arithmetic as "is the spread wide?" (retrospective §9.11.3: a repair
662
+ * must reduce the violation; the watchdog otherwise counted any placement
663
+ * as progress).
664
+ *
665
+ * @param {import('../types.js').OrderManagerLike} manager - Manager instance
666
+ * @returns {{ currentSpread: number, nominalSpread: number, excess: number, buyCount: number, sellCount: number }}
667
+ */
668
+ export declare function measureSpreadExcess(manager: OrderManagerLike): {
669
+ currentSpread: number;
670
+ nominalSpread: number;
671
+ excess: number;
672
+ buyCount: number;
673
+ sellCount: number;
674
+ };
657
675
  /**
658
676
  * Proactive spread correction check.
659
677
  *
@@ -686,6 +704,7 @@ export declare function checkSpreadCondition(manager: OrderManagerLike, _BitShar
686
704
  partialsMoved: number;
687
705
  stall?: SpreadCorrectionStall;
688
706
  transient?: boolean;
707
+ excessAfter?: number;
689
708
  }>;
690
709
  /**
691
710
  * Grid health check for structural violations.
@@ -2640,6 +2640,29 @@ export function calculateCurrentSpread(manager) {
2640
2640
  const { onChainBuys, onChainSells } = _getOnChainOrders(manager);
2641
2641
  return calculateSpreadFromOrders(onChainBuys, onChainSells);
2642
2642
  }
2643
+ /**
2644
+ * Measure the current out-of-spread violation in grid steps, from the live
2645
+ * on-chain window only. ONE implementation for the entry reading and the
2646
+ * post-placement reading, so "did the repair reduce the violation?" is the
2647
+ * same arithmetic as "is the spread wide?" (retrospective §9.11.3: a repair
2648
+ * must reduce the violation; the watchdog otherwise counted any placement
2649
+ * as progress).
2650
+ *
2651
+ * @param {import('../types.js').OrderManagerLike} manager - Manager instance
2652
+ * @returns {{ currentSpread: number, nominalSpread: number, excess: number, buyCount: number, sellCount: number }}
2653
+ */
2654
+ export function measureSpreadExcess(manager) {
2655
+ const { onChainBuys, onChainSells } = _getOnChainOrders(manager);
2656
+ const currentSpread = calculateSpreadFromOrders(onChainBuys, onChainSells);
2657
+ const nominalSpread = manager.config.targetSpreadPercent ?? DEFAULT_CONFIG.targetSpreadPercent;
2658
+ // Fixed tolerance: 0.5 steps = half increment (tighter spread check).
2659
+ const toleranceSteps = 0.5;
2660
+ // `_getOnChainOrders` already slot-N/size-gated the window.
2661
+ const buyCount = onChainBuys.length;
2662
+ const sellCount = onChainSells.length;
2663
+ const excess = shouldFlagOutOfSpread(currentSpread, nominalSpread, toleranceSteps, buyCount, sellCount, manager.config.incrementPercent);
2664
+ return { currentSpread, nominalSpread, excess, buyCount, sellCount };
2665
+ }
2643
2666
  /**
2644
2667
  * Proactive spread correction check.
2645
2668
  *
@@ -2702,21 +2725,12 @@ export async function checkSpreadCondition(manager, _BitShares, updateOrdersOnCh
2702
2725
  // activating SPREAD slots at suboptimal prices.
2703
2726
  const oneSideEmpty = onChainBuys.length === 0 || onChainSells.length === 0;
2704
2727
  const executeSpreadCheck = async () => {
2705
- const currentSpread = calculateCurrentSpread(manager);
2706
- // Nominal spread is the configured target spread percentage.
2707
- // Keep this fixed: doubled-side flags are fill/replacement mechanics only.
2708
- const nominalSpread = manager.config.targetSpreadPercent ?? DEFAULT_CONFIG.targetSpreadPercent;
2728
+ const measured = measureSpreadExcess(manager);
2729
+ const currentSpread = measured.currentSpread;
2730
+ const nominalSpread = measured.nominalSpread;
2709
2731
  // Fixed tolerance: 0.5 steps = half increment (tighter spread check).
2710
2732
  const toleranceSteps = 0.5;
2711
- const buyCount = manager.getOrdersByTypeAndState(ORDER_TYPES.BUY, ORDER_STATES.ACTIVE)
2712
- .concat(manager.getOrdersByTypeAndState(ORDER_TYPES.BUY, ORDER_STATES.PARTIAL))
2713
- .filter((o) => o?.orderId && Number(o?.size || 0) > 0 && parseSlotIndex(o?.id) !== null)
2714
- .length;
2715
- const sellCount = manager.getOrdersByTypeAndState(ORDER_TYPES.SELL, ORDER_STATES.ACTIVE)
2716
- .concat(manager.getOrdersByTypeAndState(ORDER_TYPES.SELL, ORDER_STATES.PARTIAL))
2717
- .filter((o) => o?.orderId && Number(o?.size || 0) > 0 && parseSlotIndex(o?.id) !== null)
2718
- .length;
2719
- manager.outOfSpread = shouldFlagOutOfSpread(currentSpread, nominalSpread, toleranceSteps, buyCount, sellCount, manager.config.incrementPercent);
2733
+ manager.outOfSpread = measured.excess;
2720
2734
  if (manager.outOfSpread === 0)
2721
2735
  return false;
2722
2736
  // P-A cold book: the orchestration short-circuit, evaluated in the
@@ -2985,7 +2999,19 @@ export async function checkSpreadCondition(manager, _BitShares, updateOrdersOnCh
2985
2999
  if (batchResult && batchResult.executed === true) {
2986
3000
  await manager.recalculateFunds();
2987
3001
  const placed = getSpreadCorrectionActionCount(activeCorrection);
2988
- return { ordersPlaced: placed, partialsMoved: 0 };
3002
+ // Post-placement measurement: the watchdog needs to know
3003
+ // whether this placement REDUCED the violation or merely
3004
+ // moved orders (§9.11.3). Measured from the committed
3005
+ // window, so a placement that only touched the side that
3006
+ // was not blocking reports the same excess.
3007
+ let excessAfter;
3008
+ try {
3009
+ excessAfter = measureSpreadExcess(manager).excess;
3010
+ }
3011
+ catch {
3012
+ excessAfter = undefined;
3013
+ }
3014
+ return { ordersPlaced: placed, partialsMoved: 0, ...(excessAfter === undefined ? {} : { excessAfter }) };
2989
3015
  }
2990
3016
  const cause = classifySpreadRefusalCause(batchResult);
2991
3017
  recordSpreadCorrectionRefusal(manager, cause);
@@ -108,10 +108,22 @@ declare const utils: {
108
108
  pivot: number | null;
109
109
  threshold?: number | null;
110
110
  };
111
+ isRefusedLevelRefillBlocked(manager: import("../types.js").OrderManagerLike | null | undefined, price: unknown, type: unknown): {
112
+ blocked: boolean;
113
+ level: number | null;
114
+ };
115
+ evaluateLastFillGuard(manager: import("../types.js").OrderManagerLike | null | undefined, price: number, type: unknown, pivotPrice: number | null, pivotType: string | null): {
116
+ blocked: boolean;
117
+ pivot: number | null;
118
+ threshold?: number | null;
119
+ bandExempt?: boolean;
120
+ refusedLevel?: boolean;
121
+ };
122
+ isLastFillBandSellExempt(manager: import("../types.js").OrderManagerLike | null | undefined, price: unknown): boolean;
111
123
  isLastFillPlacementBlocked(manager: import("../types.js").OrderManagerLike | null | undefined, price: unknown, type: unknown): {
112
124
  blocked: boolean;
113
125
  pivot: number | null;
114
- reason?: "direction" | "same-level";
126
+ reason?: "direction" | "same-level" | "refused-level";
115
127
  };
116
128
  readFillPipelineState(source: unknown): system.FillPipelineState;
117
129
  resolveGridUpdateBlockers(source: unknown, options?: {
@@ -145,6 +157,7 @@ declare const utils: {
145
157
  } | null;
146
158
  setLastFillPivot(manager: import("../types.js").OrderManagerLike, type: string, price: unknown, provenance: "fill" | "book", atMs?: number): boolean;
147
159
  resetLastFillPivot(manager: import("../types.js").OrderManagerLike, reason?: string): boolean;
160
+ expireLastFillGuardReferences(manager: import("../types.js").OrderManagerLike, reason?: string): void;
148
161
  restoreLastFillPivot(manager: import("../types.js").OrderManagerLike, persisted: unknown, options?: {
149
162
  now?: number;
150
163
  }): boolean;
@@ -284,6 +297,7 @@ declare const utils: {
284
297
  duplicateOrphanLogInfo: typeof order.duplicateOrphanLogInfo;
285
298
  chainOrderUnchangedFromCache: typeof order.chainOrderUnchangedFromCache;
286
299
  detectCrossedBookPlan: typeof order.detectCrossedBookPlan;
300
+ describeMarketCrossingActions: typeof order.describeMarketCrossingActions;
287
301
  collectKnownOnChainOrderIds: typeof order.collectKnownOnChainOrderIds;
288
302
  collectEdgePinnedFillIds: typeof order.collectEdgePinnedFillIds;
289
303
  edgeOrderIdSet: typeof order.edgeOrderIdSet;