dexbot 1.7.1 → 1.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/claw/package.json +1 -1
  3. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  4. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  5. package/dist/analysis/grid_correction_check.d.ts +60 -1
  6. package/dist/analysis/grid_correction_check.js +268 -18
  7. package/dist/analysis/pivot_timeline.d.ts +78 -0
  8. package/dist/analysis/pivot_timeline.js +360 -0
  9. package/dist/market_adapter/utils/file_lock.js +16 -4
  10. package/dist/modules/account_bots.d.ts +7 -0
  11. package/dist/modules/account_bots.js +7 -5
  12. package/dist/modules/bot_defaults.js +6 -1
  13. package/dist/modules/constants.d.ts +14 -0
  14. package/dist/modules/constants.js +82 -2
  15. package/dist/modules/dexbot_class.d.ts +39 -0
  16. package/dist/modules/dexbot_class.js +86 -1
  17. package/dist/modules/dexbot_cow_runtime.d.ts +31 -1
  18. package/dist/modules/dexbot_cow_runtime.js +299 -75
  19. package/dist/modules/dexbot_maintenance_runtime.d.ts +32 -2
  20. package/dist/modules/dexbot_maintenance_runtime.js +196 -16
  21. package/dist/modules/dexbot_state_recovery.js +15 -1
  22. package/dist/modules/launcher/instance_lock.d.ts +27 -0
  23. package/dist/modules/launcher/instance_lock.js +131 -0
  24. package/dist/modules/node_flap_score.d.ts +6 -6
  25. package/dist/modules/node_flap_score.js +6 -6
  26. package/dist/modules/order/accounting.d.ts +9 -4
  27. package/dist/modules/order/accounting.js +89 -26
  28. package/dist/modules/order/async_lock.d.ts +7 -1
  29. package/dist/modules/order/async_lock.js +7 -1
  30. package/dist/modules/order/export.js +25 -18
  31. package/dist/modules/order/fill_ledger.d.ts +99 -0
  32. package/dist/modules/order/fill_ledger.js +103 -0
  33. package/dist/modules/order/grid.d.ts +94 -18
  34. package/dist/modules/order/grid.js +247 -75
  35. package/dist/modules/order/grid_reconcile_internal.js +16 -4
  36. package/dist/modules/order/index.d.ts +29 -6
  37. package/dist/modules/order/index.js +5 -5
  38. package/dist/modules/order/lifecycle.js +3 -3
  39. package/dist/modules/order/manager.d.ts +36 -24
  40. package/dist/modules/order/manager.js +159 -39
  41. package/dist/modules/order/placement_lock.d.ts +30 -0
  42. package/dist/modules/order/placement_lock.js +48 -0
  43. package/dist/modules/order/sync_engine.d.ts +66 -118
  44. package/dist/modules/order/sync_engine.js +118 -162
  45. package/dist/modules/order/utils/math.d.ts +7 -3
  46. package/dist/modules/order/utils/math.js +7 -3
  47. package/dist/modules/order/utils/order.d.ts +79 -42
  48. package/dist/modules/order/utils/order.js +183 -86
  49. package/dist/modules/order/utils/system.d.ts +192 -8
  50. package/dist/modules/order/utils/system.js +534 -34
  51. package/dist/modules/paths.d.ts +1 -0
  52. package/dist/modules/paths.js +1 -0
  53. package/dist/modules/timing_policy.js +4 -2
  54. package/dist/modules/types.d.ts +49 -27
  55. package/dist/pm2.js +126 -0
  56. package/dist/unlock.js +26 -1
  57. package/docs/COW_INVARIANTS.md +5 -1
  58. package/docs/DEXBOT_COMPARISON.md +5 -5
  59. package/docs/EVOLUTION.md +10 -8
  60. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  61. package/docs/GRID_RECONCILE.md +3 -3
  62. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +192 -31
  63. package/docs/README.md +1 -1
  64. package/docs/architecture.md +26 -8
  65. package/modules/README.md +4 -0
  66. package/package.json +4 -3
@@ -265,7 +265,7 @@ declare function stopBotsConfigPollInterval(bot: BotLike): void;
265
265
  /**
266
266
  * Release the market adapter runtime for a bot.
267
267
  * In PM2 mode this is a no-op; in direct mode it calls the shared runtime's releaseBot.
268
- * @param {import('./dexbot_class.js').DEXBot} bot
268
+ * @param {unknown} _bot - Unused; release is keyed by botId.
269
269
  * @param {string} botId - Bot identifier
270
270
  * @param {string} [context='shutdown'] - Context label for logging
271
271
  * @returns {Promise<any>}
@@ -324,6 +324,27 @@ declare function checkGridLockHoldDuration(bot: BotLike): number;
324
324
  * @returns {Promise<void>}
325
325
  */
326
326
  declare function executeMaintenanceLogic(bot: BotLike, context: string): Promise<void>;
327
+ /**
328
+ * Arm a one-shot, level-triggered spread recheck (TIMING.SPREAD_STALE_RECHECK_MS).
329
+ *
330
+ * Retrospective R5.5: "Make deferred work level-triggered — never let a retry
331
+ * depend on a future event." The spread check has no driver of its own: it runs
332
+ * post-fill and on the blockchain-fetch tick (240 min by default, and the
333
+ * open-orders loop that would shorten it is OFF by default). Every watchdog
334
+ * window (warn 10 min, geometry 10 min, fund 5 min, pivot-expiry 20 min) is
335
+ * therefore unreachable while the grid sits out of tolerance, which is how a
336
+ * live bot went 86 minutes without a single check. While out of tolerance the
337
+ * watchdog now schedules its own retry, so the clocks run at any base cadence.
338
+ *
339
+ * Deduped and shutdown-safe: one timer at a time, never while a recheck is in
340
+ * flight, never while shutting down. `performPeriodicGridChecks` keeps its own
341
+ * pipeline/broadcast deferrals, so this cannot interleave with a fill cycle.
342
+ *
343
+ * @param {Object} bot - DEXBot instance
344
+ * @param {number} [delayOverrideMs] - Injectable delay (tests); production reads TIMING
345
+ * @returns {boolean} True when a recheck was armed (or already pending)
346
+ */
347
+ export declare function scheduleSpreadRecheck(bot: BotLike, delayOverrideMs?: number): boolean;
327
348
  /**
328
349
  * Out-of-spread persistence watchdog: the never-run-stale backstop.
329
350
  *
@@ -342,9 +363,12 @@ declare function executeMaintenanceLogic(bot: BotLike, context: string): Promise
342
363
  * @param {number} ordersPlaced - Correction orders placed this tick
343
364
  * @param {boolean} [transient=false] - True when the correction was refused for
344
365
  * a retryable reason (stale plan / duplicate CREATE / unclassified non-execution)
366
+ * @param {number} [excessAfter] - Out-of-spread violation (grid steps) MEASURED
367
+ * after the placement, or undefined when the correction placed nothing /
368
+ * could not be measured
345
369
  * @returns {{staleMs: number, escalated: boolean}}
346
370
  */
347
- export declare function trackOutOfSpreadStaleness(bot: BotLike, spreadChecked: boolean, ordersPlaced: number, transient?: boolean): {
371
+ export declare function trackOutOfSpreadStaleness(bot: BotLike, spreadChecked: boolean, ordersPlaced: number, transient?: boolean, excessAfter?: number): {
348
372
  staleMs: number;
349
373
  escalated: boolean;
350
374
  };
@@ -461,6 +485,11 @@ declare function getMetrics(bot: BotLike): {
461
485
  spreadCorrectionRefusedCount: number;
462
486
  spreadCorrectionRefusalCauses: Record<string, number>;
463
487
  spreadCorrectionRetryCount: number;
488
+ stalePlanDroppedCount: number;
489
+ stalePlanProceedCount: number;
490
+ clampedNoopUpdateCount: number;
491
+ chunkAbortedOpCount: number;
492
+ reloadReplacedRailCount: number;
464
493
  };
465
494
  /**
466
495
  * Read open orders from chain, sync with local state, and process any fills found.
@@ -492,6 +521,7 @@ declare const _default: {
492
521
  handlePendingTriggerReset: typeof handlePendingTriggerReset;
493
522
  setupTriggerFileDetection: typeof setupTriggerFileDetection;
494
523
  performPeriodicGridChecks: typeof performPeriodicGridChecks;
524
+ scheduleSpreadRecheck: typeof scheduleSpreadRecheck;
495
525
  isOpenOrdersSyncLoopEnabled: typeof isOpenOrdersSyncLoopEnabled;
496
526
  startOpenOrdersSyncLoop: typeof startOpenOrdersSyncLoop;
497
527
  stopOpenOrdersSyncLoop: typeof stopOpenOrdersSyncLoop;
@@ -8,7 +8,7 @@ const require = createRequire(import.meta.url);
8
8
  import { lazyShim } from './lazy_shim.js';
9
9
  import fs from 'node:fs';
10
10
  import { spawn } from 'node:child_process';
11
- import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills } from './order/utils/system.js';
11
+ import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills, resolveGridUpdateBlockers, noteGridUpdateBlockers, expireLastFillGuardReferences } from './order/utils/system.js';
12
12
  import { withPlacement } from './order/placement_lock.js';
13
13
  import { path } from './path_api.js';
14
14
  import * as chainOrders from './chain_orders.js';
@@ -740,8 +740,19 @@ function getTargetedSyncReason(bot) {
740
740
  }
741
741
  async function maybeRunTargetedDriftReconciliation(bot, context) {
742
742
  const trigger = getTargetedSyncReason(bot);
743
- if (!trigger)
743
+ if (!trigger) {
744
+ // CONSECUTIVE means consecutive: a tick that never evaluates the gate
745
+ // must clear the counter. Without this reset a counter parked at
746
+ // GRID_BLOCKED_ESCALATE_CYCLES survives an arbitrarily long
747
+ // trigger-less stretch, and the first unreadable cycle afterwards fires
748
+ // a structural resync instead of waiting the full window.
749
+ // (docs/FILL_PIPELINE.md B1; pinned by test_targeted_drift_reconcile.)
750
+ // Guarded: without a manager there is no counter to clear, and the
751
+ // helper would otherwise park a stray field on the bot.
752
+ if (bot.manager)
753
+ noteGridUpdateBlockers(bot, []);
744
754
  return false;
755
+ }
745
756
  // UNSETTLED-FILL GATE: a startup-style reconcile derives its window from the
746
757
  // master grid. While fills are still queued / draining / in flight / owed a
747
758
  // crawl, that master still holds the filled slots as ACTIVE and the boundary
@@ -752,9 +763,25 @@ async function maybeRunTargetedDriftReconciliation(bot, context) {
752
763
  // resolves against the crawled rail. No cooldown stamp, so the retry is
753
764
  // immediate once the fills settle.
754
765
  const unsettled = hasUnsettledFills(bot);
755
- if (unsettled.unsettled) {
756
- bot._log(`[TARGETED-SYNC] Deferring ${context} reconciliation: ${unsettled.reason} — ` +
757
- `the rail must absorb the fills before it is topped up`, 'debug');
766
+ const gridBlockers = resolveGridUpdateBlockers(bot);
767
+ // Escalation: only an UNREADABLE pipeline state escalates, and only after
768
+ // GRID_BLOCKED_ESCALATE_CYCLES consecutive attempts. A busy pipeline is
769
+ // normal work; the placement/broadcast watchdogs own their own bounds.
770
+ const blockedGate = noteGridUpdateBlockers(bot, gridBlockers);
771
+ if (blockedGate.escalate) {
772
+ const codes = gridBlockers.map((b) => b.code).join(',');
773
+ bot._log(`[GRID-BLOCKED] ${blockedGate.cycles} consecutive blocked cycle(s) with unreadable pipeline state ` +
774
+ `(${codes}); requesting a structural resync`, 'error');
775
+ fireStructuralResync(bot, 'grid update blocked by unreadable pipeline state', { cycles: blockedGate.cycles, codes }, '[GRID-BLOCKED]');
776
+ }
777
+ if (gridBlockers.length > 0) {
778
+ // Any blocker defers the reconcile: it is a chain-mutating update, and
779
+ // mutating while a fill batch, a broadcast or a placement region is
780
+ // unresolved is what built the stale-master plans this gate exists for.
781
+ const codes = gridBlockers.map((b) => b.code).join(',');
782
+ const reason = unsettled.reason || gridBlockers[0].detail;
783
+ bot._log(`[TARGETED-SYNC] Deferring ${context} reconciliation: ${reason} ` +
784
+ `[${codes}] — the rail must absorb the fills before it is topped up`, 'debug');
758
785
  return false;
759
786
  }
760
787
  const now = Date.now();
@@ -2074,7 +2101,7 @@ function stopBotsConfigPollInterval(bot) {
2074
2101
  /**
2075
2102
  * Release the market adapter runtime for a bot.
2076
2103
  * In PM2 mode this is a no-op; in direct mode it calls the shared runtime's releaseBot.
2077
- * @param {import('./dexbot_class.js').DEXBot} bot
2104
+ * @param {unknown} _bot - Unused; release is keyed by botId.
2078
2105
  * @param {string} botId - Bot identifier
2079
2106
  * @param {string} [context='shutdown'] - Context label for logging
2080
2107
  * @returns {Promise<any>}
@@ -2635,6 +2662,7 @@ async function executeMaintenanceLogic(bot, context) {
2635
2662
  }
2636
2663
  else {
2637
2664
  const spreadResult = await bot.manager.checkSpreadCondition(BitShares, bot.updateOrdersOnChainPlan.bind(bot));
2665
+ const spreadExcessAfter = Number(spreadResult?.excessAfter);
2638
2666
  if (await bot._abortFlowIfIllegalState(`${context} spread check`))
2639
2667
  return;
2640
2668
  // Mirror the stall cause and arm the targeted-sync refresh for
@@ -2651,7 +2679,9 @@ async function executeMaintenanceLogic(bot, context) {
2651
2679
  }
2652
2680
  // Persistence watchdog: a correction that keeps placing nothing
2653
2681
  // while the spread stays wide is a stale grid, not patience.
2654
- trackOutOfSpreadStaleness(bot, true, spreadPlaced, spreadResult?.transient === true);
2682
+ // `excessAfter` is the post-placement violation, so a placement
2683
+ // that did not reduce it no longer counts as progress.
2684
+ trackOutOfSpreadStaleness(bot, true, spreadPlaced, spreadResult?.transient === true, (spreadResult?.excessAfter !== undefined && Number.isFinite(spreadExcessAfter)) ? spreadExcessAfter : undefined);
2655
2685
  }
2656
2686
  }
2657
2687
  catch (err) {
@@ -2679,6 +2709,75 @@ function clearGeometryResyncMemory(bot) {
2679
2709
  bot._lastGeometryResyncPivotKey = '';
2680
2710
  bot._geometryResyncWarnedPivotKey = '';
2681
2711
  }
2712
+ /**
2713
+ * Arm a one-shot, level-triggered spread recheck (TIMING.SPREAD_STALE_RECHECK_MS).
2714
+ *
2715
+ * Retrospective R5.5: "Make deferred work level-triggered — never let a retry
2716
+ * depend on a future event." The spread check has no driver of its own: it runs
2717
+ * post-fill and on the blockchain-fetch tick (240 min by default, and the
2718
+ * open-orders loop that would shorten it is OFF by default). Every watchdog
2719
+ * window (warn 10 min, geometry 10 min, fund 5 min, pivot-expiry 20 min) is
2720
+ * therefore unreachable while the grid sits out of tolerance, which is how a
2721
+ * live bot went 86 minutes without a single check. While out of tolerance the
2722
+ * watchdog now schedules its own retry, so the clocks run at any base cadence.
2723
+ *
2724
+ * Deduped and shutdown-safe: one timer at a time, never while a recheck is in
2725
+ * flight, never while shutting down. `performPeriodicGridChecks` keeps its own
2726
+ * pipeline/broadcast deferrals, so this cannot interleave with a fill cycle.
2727
+ *
2728
+ * @param {Object} bot - DEXBot instance
2729
+ * @param {number} [delayOverrideMs] - Injectable delay (tests); production reads TIMING
2730
+ * @returns {boolean} True when a recheck was armed (or already pending)
2731
+ */
2732
+ export function scheduleSpreadRecheck(bot, delayOverrideMs) {
2733
+ const configured = Number.isFinite(Number(delayOverrideMs)) && Number(delayOverrideMs) > 0
2734
+ ? Number(delayOverrideMs)
2735
+ : (Number(TIMING?.SPREAD_STALE_RECHECK_MS) > 0 ? Number(TIMING.SPREAD_STALE_RECHECK_MS) : 0);
2736
+ const ms = configured > 0 ? configured : 0;
2737
+ if (!(ms > 0))
2738
+ return false;
2739
+ if (!bot || bot._shuttingDown)
2740
+ return false;
2741
+ if (bot._spreadRecheckTimer || bot._spreadRecheckInFlight)
2742
+ return true;
2743
+ try {
2744
+ bot._spreadRecheckTimer = setTimeout(async () => {
2745
+ bot._spreadRecheckTimer = null;
2746
+ if (bot._shuttingDown || bot._spreadRecheckInFlight)
2747
+ return;
2748
+ bot._spreadRecheckInFlight = true;
2749
+ try {
2750
+ bot._log(`[SPREAD-STALE] Level-triggered recheck (${Math.round(ms / 1000)}s cadence while out of tolerance)`, 'debug');
2751
+ await performPeriodicGridChecks(bot);
2752
+ }
2753
+ catch (err) {
2754
+ try {
2755
+ bot._warn(`Spread recheck failed: ${getErrorMessage(err)}`);
2756
+ }
2757
+ catch { /* logging is best-effort */ }
2758
+ }
2759
+ finally {
2760
+ bot._spreadRecheckInFlight = false;
2761
+ // `trackOutOfSpreadStaleness` runs inside the maintenance call,
2762
+ // while this flag is true, so its dedupe check intentionally
2763
+ // does not arm the next timer. Re-arm after the region ends when
2764
+ // the level is still active; otherwise a single retry would be
2765
+ // falsely documented as a recurring level-triggered repair.
2766
+ if (!bot._shuttingDown && Number(bot.manager?.outOfSpread) > 0) {
2767
+ scheduleSpreadRecheck(bot, ms);
2768
+ }
2769
+ }
2770
+ }, ms);
2771
+ const timer = bot._spreadRecheckTimer;
2772
+ if (typeof timer?.unref === 'function')
2773
+ timer.unref();
2774
+ return true;
2775
+ }
2776
+ catch {
2777
+ bot._spreadRecheckTimer = null;
2778
+ return false;
2779
+ }
2780
+ }
2682
2781
  /**
2683
2782
  * Out-of-spread persistence watchdog: the never-run-stale backstop.
2684
2783
  *
@@ -2697,19 +2796,44 @@ function clearGeometryResyncMemory(bot) {
2697
2796
  * @param {number} ordersPlaced - Correction orders placed this tick
2698
2797
  * @param {boolean} [transient=false] - True when the correction was refused for
2699
2798
  * a retryable reason (stale plan / duplicate CREATE / unclassified non-execution)
2799
+ * @param {number} [excessAfter] - Out-of-spread violation (grid steps) MEASURED
2800
+ * after the placement, or undefined when the correction placed nothing /
2801
+ * could not be measured
2700
2802
  * @returns {{staleMs: number, escalated: boolean}}
2701
2803
  */
2702
- export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, transient = false) {
2804
+ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, transient = false, excessAfter) {
2703
2805
  const outOfSpread = Number(bot?.manager?.outOfSpread) || 0;
2704
2806
  const nowMs = Date.now();
2705
2807
  const inToleranceResetMs = Number(TIMING?.SPREAD_STALE_IN_TOLERANCE_RESET_MS) > 0
2706
2808
  ? Number(TIMING.SPREAD_STALE_IN_TOLERANCE_RESET_MS)
2707
2809
  : 10 * 60 * 1000;
2708
- if (ordersPlaced > 0) {
2810
+ // PLACED IS NOT PROGRESS (retrospective §9.11.3, "a repair must reduce the
2811
+ // violation"). Clearing the clock on ANY placement let a correction that
2812
+ // only rotated the side that was not blocking report a heal every cycle:
2813
+ // live, 6 buy rotations at a still-3.0% spread reset the clock AND the
2814
+ // geometry-recenter memory, so the pivot expiry and the one-recenter-per-pivot
2815
+ // budget could never fire while the sell side stayed unquoted for 86 minutes.
2816
+ // A placement now clears the clock only when the measured violation actually
2817
+ // shrank (excessAfter < outOfSpread, or 0). Unmeasurable => legacy behavior
2818
+ // (a placement counts), so a broken measurement can never churn a re-center.
2819
+ const measured = Number(excessAfter);
2820
+ const excessKnown = excessAfter !== undefined && excessAfter !== null && Number.isFinite(measured);
2821
+ const placedReducedViolation = ordersPlaced > 0 && (!excessKnown || measured < outOfSpread);
2822
+ if (ordersPlaced > 0 && !placedReducedViolation) {
2823
+ if (!bot._outOfSpreadNoProgressWarned) {
2824
+ bot._outOfSpreadNoProgressWarned = true;
2825
+ bot._log(`[SPREAD-STALE] ${ordersPlaced} order(s) placed but the spread is still ${measured} step(s) `
2826
+ + `out of tolerance (was ${outOfSpread}) — counting as no progress, the stale clock keeps running`, 'warn');
2827
+ }
2828
+ // Fall through: the clock is NOT cleared, so the documented windows
2829
+ // (warn / geometry / fund / pivot-expiry) stay reachable.
2830
+ }
2831
+ if (placedReducedViolation) {
2709
2832
  bot._outOfSpreadSince = 0;
2710
2833
  bot._outOfSpreadStaleWarned = false;
2711
2834
  bot._outOfSpreadTransientCount = 0;
2712
2835
  bot._outOfSpreadToleranceSince = 0;
2836
+ bot._outOfSpreadNoProgressWarned = false;
2713
2837
  clearGeometryResyncMemory(bot);
2714
2838
  try {
2715
2839
  bot.manager._spreadRailOversubscribed = null;
@@ -2736,6 +2860,7 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2736
2860
  bot._outOfSpreadStaleWarned = false;
2737
2861
  bot._outOfSpreadTransientCount = 0;
2738
2862
  bot._outOfSpreadToleranceSince = 0;
2863
+ bot._outOfSpreadNoProgressWarned = false;
2739
2864
  clearGeometryResyncMemory(bot);
2740
2865
  try {
2741
2866
  bot.manager._spreadRailOversubscribed = null;
@@ -2744,6 +2869,13 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2744
2869
  return { staleMs: 0, escalated: false };
2745
2870
  }
2746
2871
  bot._outOfSpreadToleranceSince = 0;
2872
+ // LEVEL-TRIGGERED RECHECK. Every watchdog clock below assumes the spread
2873
+ // check is EVALUATED, but the base cadence is the blockchain fetch interval
2874
+ // (240 min by default) plus the post-fill path — so a grid that is out of
2875
+ // tolerance waits hours for the next look and every window becomes dead
2876
+ // code (live: 86 minutes with no check at all). While out of tolerance, arm
2877
+ // one short recheck; it re-arms itself until the spread heals.
2878
+ scheduleSpreadRecheck(bot);
2747
2879
  if (!spreadChecked) {
2748
2880
  const since = Number(bot._outOfSpreadSince) || 0;
2749
2881
  return { staleMs: since > 0 ? Date.now() - since : 0, escalated: false };
@@ -2813,12 +2945,27 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2813
2945
  const geometryEscalateMs = Number(TIMING?.SPREAD_STALE_GEOMETRY_ESCALATE_MS) > 0
2814
2946
  ? Number(TIMING.SPREAD_STALE_GEOMETRY_ESCALATE_MS)
2815
2947
  : 10 * 60 * 1000;
2948
+ // After the one re-center per pivot, the pivot gets a BOUNDED extra window
2949
+ // (SPREAD_STALE_PIVOT_EXPIRE_MS) instead of escalating to NOTHING ("a fill
2950
+ // or the pivot TTL is the only heal"). Live, a single off-market pivot buy
2951
+ // stranded the grid out of spread for the full 24h TTL because every
2952
+ // rebuild re-armed the same pivot. Past the window the guard references
2953
+ // are expired outright (expireLastFillGuardReferences) so spread
2954
+ // correction trades around the live market again; both re-arm on the next
2955
+ // real fill. 0 disables the expiry (legacy behavior).
2956
+ const pivotExpireConfigured = Number(TIMING?.SPREAD_STALE_PIVOT_EXPIRE_MS);
2957
+ const pivotExpireMs = pivotExpireConfigured > 0
2958
+ ? pivotExpireConfigured
2959
+ : pivotExpireConfigured === 0
2960
+ ? Number.POSITIVE_INFINITY
2961
+ : 20 * 60 * 1000;
2816
2962
  const escalateMs = oversubscribedFresh
2817
2963
  ? fundEscalateMs
2818
2964
  : geometryAlreadyRecentered
2819
- // Do not re-center again for the same fill. Out of the escalation
2820
- // ladder entirely: a fill (or the pivot TTL) is the only heal.
2821
- ? Number.POSITIVE_INFINITY
2965
+ // Do not re-center again for the same fill. Bounded pivot-expiry
2966
+ // window instead of Infinity: past it the guard references are
2967
+ // cleared and the market is traded around (see pivotExpireMs).
2968
+ ? pivotExpireMs
2822
2969
  : geometryBlocked
2823
2970
  ? geometryEscalateMs
2824
2971
  : (Number(TIMING?.SPREAD_STALE_ESCALATE_MS) > 0
@@ -2850,6 +2997,19 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
2850
2997
  const lastAt = Number(bot._lastSpreadStaleResyncAt) || 0;
2851
2998
  if (now - lastAt >= cooldownMs) {
2852
2999
  bot._lastSpreadStaleResyncAt = now;
3000
+ if (geometryAlreadyRecentered) {
3001
+ // The re-center for this pivot already ran and cannot clear the
3002
+ // veto; the bounded pivot-expiry window is up. Clear the guard
3003
+ // references (live + persisted) so the market is traded around
3004
+ // again instead of staying stranded until the 24h TTL.
3005
+ expireLastFillGuardReferences(bot.manager, `out of tolerance ${staleMin}min after re-centering on fill pivot ${geometryPivotKey.split('|')[0]}`);
3006
+ clearGeometryResyncMemory(bot);
3007
+ bot._outOfSpreadSince = 0;
3008
+ bot._outOfSpreadStaleWarned = false;
3009
+ bot._outOfSpreadTransientCount = 0;
3010
+ bot._outOfSpreadToleranceSince = 0;
3011
+ return { staleMs, escalated: true };
3012
+ }
2853
3013
  if (geometryBlocked && geometryPivotKey)
2854
3014
  bot._lastGeometryResyncPivotKey = geometryPivotKey;
2855
3015
  bot._log(`[SPREAD-STALE] Spread out of tolerance for ${staleMin}min despite corrections` +
@@ -3329,7 +3489,26 @@ function wireStructuralGridResyncRequest(bot) {
3329
3489
  // commit refusal + re-adoption cycle. Defer until the batch
3330
3490
  // completes; the fill consumer is already gated on _batchInFlight,
3331
3491
  // so this only blocks the recovery itself, never new fills.
3332
- if (bot._batchInFlight > 0) {
3492
+ //
3493
+ // SAME DEFER FOR AN UNSETTLED FILL PIPELINE: a fill batch mutates
3494
+ // the grid in memory long before its persistence flush reaches
3495
+ // disk, so the on-disk snapshot can still describe the PRE-fill
3496
+ // slot sizes. Reloading it rolls an applied fill back, and the next
3497
+ // fill for the same order is then subtracted from that rolled-back
3498
+ // baseline — leaving a phantom residual plus an orderId the chain
3499
+ // no longer has. Observed live: two partial fills of one order in
3500
+ // consecutive blocks; the reload between them turned a completed
3501
+ // order into a phantom parked in the gap band, which froze the
3502
+ // boundary crawl so the spread could not be closed.
3503
+ //
3504
+ // `hasUnsettledFills` is the same predicate the destructive
3505
+ // snapshot-reject gate uses: only a settled fill pipeline makes the
3506
+ // on-disk snapshot a safe base.
3507
+ const fillPipeline = hasUnsettledFills(bot);
3508
+ const deferCause = bot._batchInFlight > 0
3509
+ ? `_batchInFlight=${bot._batchInFlight}`
3510
+ : (fillPipeline.unsettled ? `fill pipeline not settled: ${fillPipeline.reason}` : null);
3511
+ if (deferCause) {
3333
3512
  const now = Date.now();
3334
3513
  if (!bot._structuralGridResyncDeferStartedAt) {
3335
3514
  bot._structuralGridResyncDeferStartedAt = now;
@@ -3338,7 +3517,7 @@ function wireStructuralGridResyncRequest(bot) {
3338
3517
  bot._structuralGridResyncDeferCount = (bot._structuralGridResyncDeferCount || 0) + 1;
3339
3518
  const deferredMs = now - bot._structuralGridResyncDeferStartedAt;
3340
3519
  if (deferredMs >= STRUCTURAL_RESYNC_MAX_DEFER_MS) {
3341
- bot._warn(`[RECOVERY] Structural resync max-defer reached (${deferredMs}ms, cap ${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms, deferred ${bot._structuralGridResyncDeferCount}x) — forcing despite _batchInFlight=${bot._batchInFlight}: ${reason}`);
3520
+ bot._warn(`[RECOVERY] Structural resync max-defer reached (${deferredMs}ms, cap ${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms, deferred ${bot._structuralGridResyncDeferCount}x) — forcing despite ${deferCause}: ${reason}`);
3342
3521
  bot._structuralGridResyncDeferStartedAt = null;
3343
3522
  bot._structuralGridResyncDeferCount = 0;
3344
3523
  }
@@ -3348,10 +3527,10 @@ function wireStructuralGridResyncRequest(bot) {
3348
3527
  // defer lines in seconds); count the rest and surface them only in
3349
3528
  // the final forced message.
3350
3529
  if (bot._structuralGridResyncDeferCount === 1) {
3351
- bot._warn(`[RECOVERY] Structural resync defer (batchInFlight=${bot._batchInFlight}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`);
3530
+ bot._warn(`[RECOVERY] Structural resync defer (${deferCause}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`);
3352
3531
  }
3353
3532
  else {
3354
- bot.manager?.logger?.log?.(`[RECOVERY] Structural resync defer (suppressed; count=${bot._structuralGridResyncDeferCount}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`, 'debug');
3533
+ bot.manager?.logger?.log?.(`[RECOVERY] Structural resync defer (suppressed; ${deferCause}, count=${bot._structuralGridResyncDeferCount}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`, 'debug');
3355
3534
  }
3356
3535
  bot._structuralGridResyncTimer = setTimeout(runStructuralResync, TIMING.LOCK_REFRESH_MIN_MS);
3357
3536
  return;
@@ -3588,6 +3767,7 @@ export default {
3588
3767
  handlePendingTriggerReset,
3589
3768
  setupTriggerFileDetection,
3590
3769
  performPeriodicGridChecks,
3770
+ scheduleSpreadRecheck,
3591
3771
  isOpenOrdersSyncLoopEnabled,
3592
3772
  startOpenOrdersSyncLoop,
3593
3773
  stopOpenOrdersSyncLoop,
@@ -11,6 +11,7 @@ import { restoreGapEvacStreaks, applyPersistedPendingCrawls, resetLastFillPivot,
11
11
  import { blockchainToFloat, calculateGapSlots, resolveGapSlots, validatePersistedBoundary, isTransientInBandRejection } from './order/utils/math.js';
12
12
  import { ON_MISSING_GENESIS, isMissingGenesisError } from './order/genesis_policy.js';
13
13
  import { hasExecutableActions } from './order/utils/validate.js';
14
+ import { releasePlanLease } from './order/placement_lock.js';
14
15
  import { getErrorMessage } from './utils/errors.js';
15
16
  const { isGridBloated } = grid;
16
17
  /**
@@ -643,7 +644,10 @@ async function _rebalanceAfterRecovery(bot, context) {
643
644
  return;
644
645
  }
645
646
  try {
646
- const rebalanceResult = await bot.manager.performSafeRebalance([], new Set());
647
+ // holdPlanRegion: every result on this path is either executed below or
648
+ // released explicitly, so the plan→commit window is serialized against
649
+ // the fill-driven planner.
650
+ const rebalanceResult = await bot.manager.performSafeRebalance([], new Set(), { holdPlanRegion: true });
647
651
  if (!rebalanceResult || rebalanceResult.aborted) {
648
652
  // A deferred abort means owed work is still outstanding (broadcast
649
653
  // held the plan, or the identical-held-plan suppression is waiting
@@ -661,9 +665,19 @@ async function _rebalanceAfterRecovery(bot, context) {
661
665
  // rebalance stack stays balanced and _rebalanceState returns to
662
666
  // NORMAL (mirrors _executeBatchIfNeeded's no-action pop). Aborted
663
667
  // results are never pushed, so only this branch needs the pop.
668
+ // The result also owns the plan region acquired before planning;
669
+ // no executor will adopt it on this path, so release it here.
670
+ releasePlanLease(rebalanceResult);
664
671
  bot.manager?._popWorkingGridRef?.(rebalanceResult);
665
672
  return;
666
673
  }
674
+ const railCount = Array.isArray(rebalanceResult.actions) ? rebalanceResult.actions.length : 0;
675
+ if (railCount > 0 && bot._metrics) {
676
+ // Cost of a recovery reload/repair in re-placed rails. Counted here
677
+ // (not derived from logs) so the reload→re-place amplification is
678
+ // observable next to the other churn counters.
679
+ bot._metrics.reloadReplacedRailCount = (bot._metrics.reloadReplacedRailCount || 0) + railCount;
680
+ }
667
681
  bot.manager.logger?.log?.(`[RECOVERY] Re-placing ${(rebalanceResult.actions ?? []).length} missing rail(s) after ${context}`, 'info');
668
682
  await bot.updateOrdersOnChainBatch(rebalanceResult);
669
683
  }
@@ -0,0 +1,27 @@
1
+ /** A held instance lock. release() is ownership-checked and idempotent. */
2
+ export interface BotInstanceLock {
3
+ release(): void;
4
+ /** Absolute path of the lock file (diagnostics/tests). */
5
+ lockPath: string;
6
+ }
7
+ export interface AcquireInstanceLockOptions {
8
+ /** Best-effort logger (manager logger or launcher console shim). */
9
+ log?: (message: string, level?: string) => void;
10
+ }
11
+ /**
12
+ * Acquire the single-controller lock for `account`.
13
+ *
14
+ * Throws (never waits) when another LIVE process holds the lock — the error
15
+ * message names the holder (pid, bot, start time) so the operator can stop the
16
+ * right instance. A DEAD holder (crashed process, recycled pid) is detected
17
+ * via /proc liveness + starttime match and its stale lock is taken over
18
+ * immediately, so a PM2 restart after a crash is not blocked by its own
19
+ * previous incarnation.
20
+ *
21
+ * @param {string} account - Chain account name (lock key)
22
+ * @param {string} botName - Configured bot name (diagnostics only)
23
+ * @param {Object} [options]
24
+ * @returns {BotInstanceLock} Handle with release()
25
+ */
26
+ export declare function acquireBotInstanceLock(account: string, botName?: string, options?: AcquireInstanceLockOptions): BotInstanceLock;
27
+ //# sourceMappingURL=instance_lock.d.ts.map
@@ -0,0 +1,131 @@
1
+ 'use strict';
2
+ // node-only entry point — per-account bot instance lock (/proc liveness, fs)
3
+ /**
4
+ * instance_lock.ts - Single-controller lock per trading account
5
+ *
6
+ * Prevents two bot processes (e.g. one started via `dexbot pm2` and one via a
7
+ * normal `dexbot start` unlock) from controlling the SAME account at the same
8
+ * time. Observed live (2026-11-14): two controllers kept independent in-memory
9
+ * grids over the same 40 chain orders, cancelled each other's orders as
10
+ * "duplicate orphans", attributed the same fill to different slots, and
11
+ * finally stranded the grid out of spread for ~24h behind an off-market
12
+ * LAST-FILL-GUARD pivot. This lock makes the second controller fail fast with
13
+ * the holder's identity instead of silently double-trading.
14
+ *
15
+ * The lock is keyed by ACCOUNT, not bot name: the conflict domain is the chain
16
+ * account's open orders and fills, so even two differently named bot entries
17
+ * sharing an account are refused.
18
+ *
19
+ * Implementation reuses the market adapter's file-lock primitive (staleness +
20
+ * heartbeat + ownership-checked release) with a process-liveness aliveCheck
21
+ * that also compares /proc/<pid>/stat starttime, so a recycled pid can neither
22
+ * keep a dead holder's lock alive nor steal a live holder's lock.
23
+ */
24
+ import { path } from '../path_api.js';
25
+ import { getStorage } from '../storage/index.js';
26
+ import { PATHS } from '../paths.js';
27
+ import { runtime } from '../runtime.js';
28
+ import { getProcessDiscovery } from '../process_discovery.js';
29
+ import { acquireFileLockSync, releaseFileLockSync } from '../../market_adapter/utils/file_lock.js';
30
+ import { getErrorMessage } from '../utils/errors.js';
31
+ const storage = getStorage();
32
+ function sanitizeLockKey(raw) {
33
+ return String(raw || 'unknown').toLowerCase().replace(/[^a-z0-9._-]+/g, '_').slice(0, 64) || 'unknown';
34
+ }
35
+ function instanceLocksDir() {
36
+ return PATHS.PROFILES.INSTANCE_LOCKS_DIR || path.join(PATHS.PROFILES_DIR, 'instance-locks');
37
+ }
38
+ function lockPathForAccount(account) {
39
+ return path.join(instanceLocksDir(), `${sanitizeLockKey(account)}.lock`);
40
+ }
41
+ function readLockPayload(lockPath) {
42
+ try {
43
+ const parsed = storage.readJSON(lockPath);
44
+ return parsed && typeof parsed === 'object' ? parsed : null;
45
+ }
46
+ catch (_) {
47
+ return null;
48
+ }
49
+ }
50
+ /**
51
+ * Acquire the single-controller lock for `account`.
52
+ *
53
+ * Throws (never waits) when another LIVE process holds the lock — the error
54
+ * message names the holder (pid, bot, start time) so the operator can stop the
55
+ * right instance. A DEAD holder (crashed process, recycled pid) is detected
56
+ * via /proc liveness + starttime match and its stale lock is taken over
57
+ * immediately, so a PM2 restart after a crash is not blocked by its own
58
+ * previous incarnation.
59
+ *
60
+ * @param {string} account - Chain account name (lock key)
61
+ * @param {string} botName - Configured bot name (diagnostics only)
62
+ * @param {Object} [options]
63
+ * @returns {BotInstanceLock} Handle with release()
64
+ */
65
+ export function acquireBotInstanceLock(account, botName = '', options = {}) {
66
+ const lockPath = lockPathForAccount(account);
67
+ const discovery = getProcessDiscovery();
68
+ try {
69
+ storage.ensureDir(instanceLocksDir());
70
+ }
71
+ catch (err) {
72
+ throw new Error(`Cannot create instance-lock directory ${instanceLocksDir()}: ${getErrorMessage(err)}`);
73
+ }
74
+ const buildPayload = () => ({
75
+ pid: runtime.pid,
76
+ botName: String(botName || ''),
77
+ account: String(account || ''),
78
+ starttime: discovery.readStat(runtime.pid)?.starttime ?? 0,
79
+ at: new Date().toISOString(),
80
+ });
81
+ const aliveCheck = (pid) => {
82
+ if (!discovery.isAlive(pid))
83
+ return false;
84
+ // pid-recycle guard: the payload's starttime is the holder's boot
85
+ // identity. A recycled pid whose /proc starttime differs is NOT the
86
+ // holder — treat the lock as dead so a new controller can take over.
87
+ const info = readLockPayload(lockPath);
88
+ const holderStart = Number(info?.starttime);
89
+ if (Number(info?.pid) === pid && holderStart > 0) {
90
+ const stat = discovery.readStat(pid);
91
+ if (stat && stat.starttime !== holderStart)
92
+ return false;
93
+ }
94
+ return true;
95
+ };
96
+ const alreadyRunningMsg = () => {
97
+ const info = readLockPayload(lockPath);
98
+ const holderBot = info?.botName ? ` (bot ${info.botName})` : '';
99
+ const holderAt = info?.at ? `, started ${info.at}` : '';
100
+ return (`account '${account}' is already controlled by another bot process ` +
101
+ `(pid ${info?.pid ?? 'unknown'}${holderBot}${holderAt}, lock: ${lockPath}). ` +
102
+ `Stop that instance first (dexbot pm2 stop / dexbot stop) — two controllers ` +
103
+ `on one account corrupt each other's grids and fills.`);
104
+ };
105
+ const lock = acquireFileLockSync(lockPath, {
106
+ // Fail fast — the second controller must not wait out a live holder.
107
+ contention: 'throw',
108
+ // Only bounds an ALIVE-but-never-beating holder (heartbeat stopped
109
+ // without process death); a dead holder is stolen immediately by the
110
+ // aliveCheck above regardless of staleness.
111
+ staleMs: 10 * 60 * 1000,
112
+ aliveCheck,
113
+ buildPayload,
114
+ alreadyRunningMsg,
115
+ timeoutMsg: `cannot acquire instance lock for account '${account}': ${lockPath}`,
116
+ });
117
+ options.log?.(`Instance lock acquired for account '${account}' (bot ${botName || 'n/a'}, pid ${runtime.pid})`, 'info');
118
+ return {
119
+ lockPath,
120
+ release() {
121
+ try {
122
+ releaseFileLockSync({ fd: lock.fd, lockPath: lock.lockPath, heartbeat: lock.heartbeat, ownerId: lock.ownerId });
123
+ options.log?.(`Instance lock released for account '${account}'`, 'info');
124
+ }
125
+ catch (err) {
126
+ options.log?.(`Instance lock release failed for account '${account}': ${getErrorMessage(err)}`, 'warn');
127
+ }
128
+ },
129
+ };
130
+ }
131
+ //# sourceMappingURL=instance_lock.js.map
@@ -11,13 +11,13 @@
11
11
  *
12
12
  * effectiveLatency = latencyMs + flapPenaltyMs
13
13
  * flapScore = Σ 0.5 ^ (age_i / halfLife)
14
- * flapPenaltyMs = min(flapScore * penaltyPerDrop, maxPenaltyMs)
14
+ * flapPenaltyMs = min(flapScore * DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_PENALTY_MAX_MS)
15
15
  *
16
- * A node with no recent drops ranks on its ping alone. One drop costs
17
- * `penaltyPerDrop` of effective latency and then fades with `halfLife`, so a
18
- * node that flaps once and then behaves is re-admitted automatically, a node
19
- * that keeps flapping stays at the back of the list, and nothing has to be
20
- * "reset" by a lucky health probe (the old
16
+ * A node with no recent drops ranks on its ping alone. One undecayed drop costs
17
+ * DEFAULT_FLAP_PENALTY_MS_PER_DROP of effective latency and then fades with the
18
+ * half-life, so a node that flaps once and then behaves is re-admitted
19
+ * automatically, a node that keeps flapping stays at the back of the list, and
20
+ * nothing has to be "reset" by a lucky health probe (the old
21
21
  * LIVE_FAILURE_HEALTH_SUCCESS_STREAK wipe-on-success counter).
22
22
  *
23
23
  * Everything here is pure: callers own persistence and logging. Drop history
@@ -12,13 +12,13 @@
12
12
  *
13
13
  * effectiveLatency = latencyMs + flapPenaltyMs
14
14
  * flapScore = Σ 0.5 ^ (age_i / halfLife)
15
- * flapPenaltyMs = min(flapScore * penaltyPerDrop, maxPenaltyMs)
15
+ * flapPenaltyMs = min(flapScore * DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_PENALTY_MAX_MS)
16
16
  *
17
- * A node with no recent drops ranks on its ping alone. One drop costs
18
- * `penaltyPerDrop` of effective latency and then fades with `halfLife`, so a
19
- * node that flaps once and then behaves is re-admitted automatically, a node
20
- * that keeps flapping stays at the back of the list, and nothing has to be
21
- * "reset" by a lucky health probe (the old
17
+ * A node with no recent drops ranks on its ping alone. One undecayed drop costs
18
+ * DEFAULT_FLAP_PENALTY_MS_PER_DROP of effective latency and then fades with the
19
+ * half-life, so a node that flaps once and then behaves is re-admitted
20
+ * automatically, a node that keeps flapping stays at the back of the list, and
21
+ * nothing has to be "reset" by a lucky health probe (the old
22
22
  * LIVE_FAILURE_HEALTH_SUCCESS_STREAK wipe-on-success counter).
23
23
  *
24
24
  * Everything here is pure: callers own persistence and logging. Drop history