dexbot 1.7.1 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/grid_correction_check.d.ts +60 -1
- package/dist/analysis/grid_correction_check.js +268 -18
- package/dist/analysis/pivot_timeline.d.ts +78 -0
- package/dist/analysis/pivot_timeline.js +360 -0
- package/dist/market_adapter/utils/file_lock.js +16 -4
- package/dist/modules/account_bots.d.ts +7 -0
- package/dist/modules/account_bots.js +7 -5
- package/dist/modules/bot_defaults.js +6 -1
- package/dist/modules/constants.d.ts +14 -0
- package/dist/modules/constants.js +82 -2
- package/dist/modules/dexbot_class.d.ts +39 -0
- package/dist/modules/dexbot_class.js +86 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +31 -1
- package/dist/modules/dexbot_cow_runtime.js +299 -75
- package/dist/modules/dexbot_maintenance_runtime.d.ts +32 -2
- package/dist/modules/dexbot_maintenance_runtime.js +196 -16
- package/dist/modules/dexbot_state_recovery.js +15 -1
- package/dist/modules/launcher/instance_lock.d.ts +27 -0
- package/dist/modules/launcher/instance_lock.js +131 -0
- package/dist/modules/node_flap_score.d.ts +6 -6
- package/dist/modules/node_flap_score.js +6 -6
- package/dist/modules/order/accounting.d.ts +9 -4
- package/dist/modules/order/accounting.js +89 -26
- package/dist/modules/order/async_lock.d.ts +7 -1
- package/dist/modules/order/async_lock.js +7 -1
- package/dist/modules/order/export.js +25 -18
- package/dist/modules/order/fill_ledger.d.ts +99 -0
- package/dist/modules/order/fill_ledger.js +103 -0
- package/dist/modules/order/grid.d.ts +94 -18
- package/dist/modules/order/grid.js +247 -75
- package/dist/modules/order/grid_reconcile_internal.js +16 -4
- package/dist/modules/order/index.d.ts +29 -6
- package/dist/modules/order/index.js +5 -5
- package/dist/modules/order/lifecycle.js +3 -3
- package/dist/modules/order/manager.d.ts +36 -24
- package/dist/modules/order/manager.js +159 -39
- package/dist/modules/order/placement_lock.d.ts +30 -0
- package/dist/modules/order/placement_lock.js +48 -0
- package/dist/modules/order/sync_engine.d.ts +66 -118
- package/dist/modules/order/sync_engine.js +118 -162
- package/dist/modules/order/utils/math.d.ts +7 -3
- package/dist/modules/order/utils/math.js +7 -3
- package/dist/modules/order/utils/order.d.ts +79 -42
- package/dist/modules/order/utils/order.js +183 -86
- package/dist/modules/order/utils/system.d.ts +192 -8
- package/dist/modules/order/utils/system.js +534 -34
- package/dist/modules/paths.d.ts +1 -0
- package/dist/modules/paths.js +1 -0
- package/dist/modules/timing_policy.js +4 -2
- package/dist/modules/types.d.ts +49 -27
- package/dist/pm2.js +126 -0
- package/dist/unlock.js +26 -1
- package/docs/COW_INVARIANTS.md +5 -1
- package/docs/DEXBOT_COMPARISON.md +5 -5
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_RECONCILE.md +3 -3
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +192 -31
- package/docs/README.md +1 -1
- package/docs/architecture.md +26 -8
- package/modules/README.md +4 -0
- package/package.json +4 -3
|
@@ -235,7 +235,12 @@ let TIMING = {
|
|
|
235
235
|
// BTS acquisition for non-BTS pairs: min cooldown between acquisition attempts (minutes)
|
|
236
236
|
BTS_ACQUIRE_COOLDOWN_MIN: 60,
|
|
237
237
|
// Fill processing timing
|
|
238
|
-
|
|
238
|
+
// Claim LEASE for a fill accepted into the queue but not yet booked
|
|
239
|
+
// (deferral, crash). It is NOT the dedupe authority: the processed-fill
|
|
240
|
+
// store refuses a booked key while it is retained, regardless of age;
|
|
241
|
+
// FILL_RECORD_RETENTION_MS bounds that retention, it is not an expiry
|
|
242
|
+
// (docs/FILL_PIPELINE.md stage 2).
|
|
243
|
+
FILL_DEDUPE_WINDOW_MS: 5000,
|
|
239
244
|
FILL_RECORD_RETENTION_MS: 7 * 24 * 60 * 60 * 1000, // 7 days - how long to keep persisted fill records (was 1h, increased to close double-credit window on restarts >1h apart)
|
|
240
245
|
PROCESSED_FILL_PERSIST_BATCH_MS: 250, // 250ms - coalesce processed-fill persistence writes under burst load
|
|
241
246
|
PROCESSED_FILL_PERSIST_BATCH_SIZE: 25, // Flush immediately once this many processed fills are queued
|
|
@@ -400,6 +405,16 @@ let TIMING = {
|
|
|
400
405
|
// (`_lastGeometryResyncPivotKey`): the pivot does not move with the grid, so
|
|
401
406
|
// a second attempt re-derives the identical veto.
|
|
402
407
|
SPREAD_STALE_GEOMETRY_ESCALATE_MS: 10 * 60 * 1000, // 10 minutes
|
|
408
|
+
// SPREAD_STALE_PIVOT_EXPIRE_MS: how long after the ONE geometry re-center
|
|
409
|
+
// per pivot the out-of-spread watchdog waits before clearing the
|
|
410
|
+
// LAST-FILL-GUARD pivot (and the bot-scoped last-market-fill reference)
|
|
411
|
+
// outright. Previously that state escalated to NOTHING ("a fill or the
|
|
412
|
+
// pivot TTL is the only heal") — live, a single off-market pivot buy
|
|
413
|
+
// stranded the grid out of spread for the full 24h TTL because every
|
|
414
|
+
// rebuild re-armed the same pivot. Clearing both references lets spread
|
|
415
|
+
// correction trade around the live market again; both re-arm on the next
|
|
416
|
+
// real fill. 0 disables the expiry (legacy behavior).
|
|
417
|
+
SPREAD_STALE_PIVOT_EXPIRE_MS: 20 * 60 * 1000, // 20 minutes
|
|
403
418
|
// SPREAD_CORRECTION_MAX_ATTEMPTS: bounded attempts for one spread-correction
|
|
404
419
|
// cycle (initial broadcast + fresh-master re-plan retries). Kept next to the
|
|
405
420
|
// other SPREAD_* tunables so the retry budget is centralized; a value < 1
|
|
@@ -425,6 +440,17 @@ let TIMING = {
|
|
|
425
440
|
// separate from BOUNDARY_HOLD_RESYNC_COOLDOWN_MS so tuning boundary-hold
|
|
426
441
|
// behavior never silently changes the spread watchdog cadence.
|
|
427
442
|
SPREAD_STALE_RESYNC_COOLDOWN_MS: 5 * 60 * 1000, // 5 minutes
|
|
443
|
+
// SPREAD_STALE_RECHECK_MS: level-triggered retry cadence for the spread
|
|
444
|
+
// watchdog. Every clock above (5/10/20/30 min) assumes the spread check is
|
|
445
|
+
// EVALUATED, but with OPEN_ORDERS_SYNC_LOOP_ENABLED=false the only driver
|
|
446
|
+
// is the post-fill path plus the BLOCKCHAIN_FETCH_INTERVAL_MIN (240 min)
|
|
447
|
+
// tick — so an out-of-tolerance grid was re-checked once per 4h and every
|
|
448
|
+
// window above was dead code (live: 86 minutes with no check at all, a
|
|
449
|
+
// second identical batch accepted only when the mid finally moved). While
|
|
450
|
+
// the spread is out of tolerance the watchdog now arms a one-shot recheck
|
|
451
|
+
// at this cadence, so the documented clocks are reachable at any base
|
|
452
|
+
// cadence. 0 disables the retry (legacy cadence-only behavior).
|
|
453
|
+
SPREAD_STALE_RECHECK_MS: 60 * 1000, // 1 minute
|
|
428
454
|
// BOUNDARY_HOLD_RESYNC_THRESHOLD / COOLDOWN: consecutive boundary-hold
|
|
429
455
|
// batches (each carrying fresh fills) after which the COW executor asks
|
|
430
456
|
// for a guard-aware structural re-center. A hold is correct maker
|
|
@@ -773,6 +799,51 @@ let GRID_LIMITS = {
|
|
|
773
799
|
// runtime are NOT expired — the guard has no clock there and the pivot
|
|
774
800
|
// is refreshed by every fresh fill.
|
|
775
801
|
LAST_FILL_PIVOT_TTL_MS: 24 * 60 * 60 * 1000, // 24 hours
|
|
802
|
+
// LAST_FILL_REFUSED_LEVEL_TTL_MS: how long a fill REFUSED from arming the
|
|
803
|
+
// pivot (wrong-side / too-far premium sweep, §9.12/P1) still marks its own
|
|
804
|
+
// level as a same-level re-fill block. The direction pivot deliberately
|
|
805
|
+
// stays behind such a fill — but the §9.12/P2 same-level refusal keys off
|
|
806
|
+
// the ARMED pivot, so the P1 refusal silently suspended it: a spread-
|
|
807
|
+
// correction slot re-placed at the just-filled level ("rises" of sub-sat
|
|
808
|
+
// rounding dust) refilled 5 times in 2.5 minutes on a live market-pair bot
|
|
809
|
+
// (2026-11-16). The marker vetoes exactly ONE level — the just-filled one —
|
|
810
|
+
// and nothing else, so spread correction keeps trading around the live
|
|
811
|
+
// market and the unquoted-side deadlock P1 was written against cannot
|
|
812
|
+
// re-open through it. In-memory only and short-lived: it must stop the
|
|
813
|
+
// immediate re-fill loop, not ratchet for the pivot's 24h TTL.
|
|
814
|
+
LAST_FILL_REFUSED_LEVEL_TTL_MS: 30 * 60 * 1000, // 30 minutes
|
|
815
|
+
// LAST_FILL_PIVOT_WRONG_SIDE_TOLERANCE_STEPS: a real FILL may only arm the
|
|
816
|
+
// LAST-FILL-GUARD pivot when it priced on the market's own side of the
|
|
817
|
+
// live reference: BUY at/below mid, SELL at/above mid (within this
|
|
818
|
+
// tolerance in grid increments). A fill on the WRONG side (a BUY above
|
|
819
|
+
// mid, a SELL below mid) is a crossed-book artifact — usually a misplaced
|
|
820
|
+
// order that got swept — and arming it vetoed every near-market SELL for
|
|
821
|
+
// the full 24h TTL in a live incident. The pivot AND the bot-scoped
|
|
822
|
+
// last-market-fill row are both skipped with the refusal (an off-mid row
|
|
823
|
+
// as a crossing floor freezes correction the same way), so the last
|
|
824
|
+
// honest fill stays the reference. Fail-open when no fresh live price
|
|
825
|
+
// exists.
|
|
826
|
+
LAST_FILL_PIVOT_WRONG_SIDE_TOLERANCE_STEPS: 1,
|
|
827
|
+
// LAST_FILL_PIVOT_MAX_MID_DISTANCE_STEPS: a real FILL may also only arm the
|
|
828
|
+
// LAST-FILL-GUARD pivot when it priced within this many grid increments of
|
|
829
|
+
// the live reference, on EITHER side. The wrong-side tolerance below catches
|
|
830
|
+
// the direction; this catches a premium SWEEP of our own rail — a taker
|
|
831
|
+
// paying through several of our levels is on the "honest" side by direction
|
|
832
|
+
// but is not a window trade. Live: a sell 14.5 steps (4.4%) above the mid
|
|
833
|
+
// armed the pivot at the top of the gap band the geometry had just derived,
|
|
834
|
+
// so every band sell was vetoed and the spread stayed 3x wide for 86 min.
|
|
835
|
+
// One full target spread is ~5 steps at the default increment/spread, so 6
|
|
836
|
+
// keeps every interior window fill and refuses a fill a full spread away.
|
|
837
|
+
// 0 disables the distance leg (direction bound only).
|
|
838
|
+
LAST_FILL_PIVOT_MAX_MID_DISTANCE_STEPS: 6,
|
|
839
|
+
// LAST_FILL_PIVOT_MARKET_FRESH_MS: how old the cached live market price
|
|
840
|
+
// (manager._liveMarketPrice) may be for the pivot market bound above to
|
|
841
|
+
// trust it. Older than this, the check fails open — an ancient mid must
|
|
842
|
+
// never veto a fresh fill after a fast market move. (The crossing FLOOR
|
|
843
|
+
// does not use this window: resolveSpreadCrossingFloor already picks the
|
|
844
|
+
// newer of the live read and the fill, and the pre-broadcast crossing
|
|
845
|
+
// check is an observation, not a veto — see retrospective §9.10/P4.)
|
|
846
|
+
LAST_FILL_PIVOT_MARKET_FRESH_MS: 15 * 60 * 1000, // 15 minutes
|
|
776
847
|
// Grid comparison metrics
|
|
777
848
|
// Detects significant divergence between calculated (in-memory) and persisted grid state
|
|
778
849
|
// after order fills and rotations
|
|
@@ -955,8 +1026,17 @@ let FILL_PROCESSING = {
|
|
|
955
1026
|
// into gapSlots+1-op transactions, so this only removes the intermediate
|
|
956
1027
|
// plan/broadcast cycles (Stale-Restart #5). Default OFF: opt-in rollout
|
|
957
1028
|
// for the live boundary engine, overridable per bot via
|
|
958
|
-
// `fillProcessing.COALESCE_BACKLOG_REPLAY` in
|
|
1029
|
+
// `fillProcessing.COALESCE_BACKLOG_REPLAY` in bots.json (or globally via
|
|
1030
|
+
// FILL_PROCESSING in general.settings.json).
|
|
959
1031
|
COALESCE_BACKLOG_REPLAY: false,
|
|
1032
|
+
// GRID_BLOCKED_ESCALATE_CYCLES: how many consecutive attempts a grid update
|
|
1033
|
+
// may be refused with an UNREADABLE pipeline state before the blocker
|
|
1034
|
+
// escalation fires a structural resync. A cycle count, not a timing window:
|
|
1035
|
+
// it is deliberately not a TIMING entry (see modules/timing_policy.ts) and
|
|
1036
|
+
// must not be turned into one. Only `unknown-blocker` escalates — a busy
|
|
1037
|
+
// pipeline (fills in flight, an unresolved broadcast, a held placement
|
|
1038
|
+
// region) is normal work and is bounded by its own watchdog.
|
|
1039
|
+
GRID_BLOCKED_ESCALATE_CYCLES: 12,
|
|
960
1040
|
// MAX_CONSECUTIVE_CONSUMER_FAILURES: Threshold for the _consumeFillQueue
|
|
961
1041
|
// watchdog. Below this count, the consumer re-schedules on every failure
|
|
962
1042
|
// via setImmediate. At or above this count, the consumer switches to
|
|
@@ -70,6 +70,7 @@ import cowRuntime from './dexbot_cow_runtime.js';
|
|
|
70
70
|
import type { ChainOrderLike, CowResult, OpPhase } from './dexbot_cow_runtime.js';
|
|
71
71
|
import { normalizeBotEntry } from './bot_settings.js';
|
|
72
72
|
import type { FillOpLike, FillLike, ReplaySafeFillAccountingOptions } from './dexbot_fill_runtime.js';
|
|
73
|
+
import { type BotInstanceLock } from './launcher/instance_lock.js';
|
|
73
74
|
import type { BotLike, AssetPair, IncomingFill, UnknownRecord, GridConfig, OrderManagerLike, AccountOrdersLike, BotMetrics, ProcessedFillStoreLike, CreditRuntimeLike, FillBatchResult, SigningToken, SpreadCorrectionStall } from './types.js';
|
|
74
75
|
/**
|
|
75
76
|
* Resolve the COALESCE_BACKLOG_REPLAY opt-in.
|
|
@@ -196,9 +197,20 @@ declare class DEXBot implements BotLike {
|
|
|
196
197
|
_marketAdapterWatchdogInFlight: boolean;
|
|
197
198
|
_outOfSpreadSince: number;
|
|
198
199
|
_outOfSpreadStaleWarned: boolean;
|
|
200
|
+
/** Held single-controller lock for this bot's account (null when not started). */
|
|
201
|
+
_instanceLock: BotInstanceLock | null;
|
|
199
202
|
_outOfSpreadTransientCount: number;
|
|
200
203
|
/** When the spread first read within tolerance; cleared on any out-of-tolerance cycle. */
|
|
201
204
|
_outOfSpreadToleranceSince: number;
|
|
205
|
+
/**
|
|
206
|
+
* Level-triggered spread recheck (see TIMING.SPREAD_STALE_RECHECK_MS).
|
|
207
|
+
* Armed by the out-of-spread watchdog while the grid is out of tolerance,
|
|
208
|
+
* since the base maintenance cadence can be hours.
|
|
209
|
+
*/
|
|
210
|
+
_spreadRecheckTimer: ReturnType<typeof setTimeout> | null;
|
|
211
|
+
_spreadRecheckInFlight: boolean;
|
|
212
|
+
/** "placed but did not reduce the violation" was already reported for this run. */
|
|
213
|
+
_outOfSpreadNoProgressWarned: boolean;
|
|
202
214
|
_postRecoveryRebalanceDefers: number;
|
|
203
215
|
_skipEmptyReadConfirmDelay: boolean;
|
|
204
216
|
_spreadStallArmed: boolean;
|
|
@@ -421,6 +433,28 @@ declare class DEXBot implements BotLike {
|
|
|
421
433
|
* @returns {string|null} Fallback key or null
|
|
422
434
|
*/
|
|
423
435
|
_buildOrphanFillFallbackKey(fill: FillLike): string | null;
|
|
436
|
+
/**
|
|
437
|
+
* Claim a fill key for this cycle, or report that it was already handled.
|
|
438
|
+
*
|
|
439
|
+
* Two authorities, in this order:
|
|
440
|
+
* 1. **The processed-fill store** — booked keys. Checked first and without
|
|
441
|
+
* any clock, because "already credited" does not expire. Before P2 a
|
|
442
|
+
* booked key older than the lease window was accepted again here and
|
|
443
|
+
* only refused later by accounting, so the replay still paid for queue
|
|
444
|
+
* churn and a FILL DETECTED log.
|
|
445
|
+
* 2. **The claim lease** — fills accepted but not yet booked (deferral,
|
|
446
|
+
* crash). Its TTL (`FILL_DEDUPE_WINDOW_MS`) is a lease, not the dedupe
|
|
447
|
+
* authority; the store refuses a booked key while retained, at any age.
|
|
448
|
+
*
|
|
449
|
+
* Claims are released on every path that does not book (requeue, orphan
|
|
450
|
+
* adoption, stale-totals deferral), so a held claim means "in flight".
|
|
451
|
+
*
|
|
452
|
+
* @param {string} fillKey - Chain fill identity (`orderId:blockNum:historyId`)
|
|
453
|
+
* @param {Set<string>} processedFillKeys - Per-cycle key set
|
|
454
|
+
* @param {string} [label] - Log prefix
|
|
455
|
+
* @param {string} [orderId] - Chain order id, for the log only
|
|
456
|
+
* @returns {boolean} true when this cycle may process the fill
|
|
457
|
+
*/
|
|
424
458
|
_isNewFillKey(fillKey: string, processedFillKeys: Set<string>, label?: string, orderId?: string): boolean;
|
|
425
459
|
/**
|
|
426
460
|
* Apply replay-safe fill accounting for tracked fills (those with a valid grid order).
|
|
@@ -1031,6 +1065,11 @@ declare class DEXBot implements BotLike {
|
|
|
1031
1065
|
spreadCorrectionRefusedCount: number;
|
|
1032
1066
|
spreadCorrectionRefusalCauses: Record<string, number>;
|
|
1033
1067
|
spreadCorrectionRetryCount: number;
|
|
1068
|
+
stalePlanDroppedCount: number;
|
|
1069
|
+
stalePlanProceedCount: number;
|
|
1070
|
+
clampedNoopUpdateCount: number;
|
|
1071
|
+
chunkAbortedOpCount: number;
|
|
1072
|
+
reloadReplacedRailCount: number;
|
|
1034
1073
|
};
|
|
1035
1074
|
/**
|
|
1036
1075
|
* Execute grid maintenance checks in strict order with pipeline consensus.
|
|
@@ -82,11 +82,14 @@ import { PATHS } from './paths.js';
|
|
|
82
82
|
import { path } from './path_api.js';
|
|
83
83
|
import * as Format from './order/format.js';
|
|
84
84
|
import cowRuntime from './dexbot_cow_runtime.js';
|
|
85
|
+
import { takePlanLease, releasePlanLease } from './order/placement_lock.js';
|
|
85
86
|
import { ProcessedFillStore } from './order/processed_fill_store.js';
|
|
87
|
+
import { isFillBooked } from './order/fill_ledger.js';
|
|
86
88
|
import { TIMING, DAEMON_CODES, FILL_PROCESSING, } from './constants.js';
|
|
87
89
|
import { normalizeBotEntry } from './bot_settings.js';
|
|
88
90
|
import { createDefaultBotMetrics } from './bot_defaults.js';
|
|
89
91
|
import { getErrorMessage, getErrorCode } from './utils/errors.js';
|
|
92
|
+
import { acquireBotInstanceLock } from './launcher/instance_lock.js';
|
|
90
93
|
const waitForConnected = lazyShim('./bitshares_client', 'waitForConnected');
|
|
91
94
|
const getKeyStore = lazyShim('./key_store', 'getKeyStore');
|
|
92
95
|
const hasExecutableActions = lazyShim('./order/utils/validate', 'hasExecutableActions');
|
|
@@ -224,9 +227,20 @@ class DEXBot {
|
|
|
224
227
|
_marketAdapterWatchdogInFlight;
|
|
225
228
|
_outOfSpreadSince;
|
|
226
229
|
_outOfSpreadStaleWarned;
|
|
230
|
+
/** Held single-controller lock for this bot's account (null when not started). */
|
|
231
|
+
_instanceLock;
|
|
227
232
|
_outOfSpreadTransientCount;
|
|
228
233
|
/** When the spread first read within tolerance; cleared on any out-of-tolerance cycle. */
|
|
229
234
|
_outOfSpreadToleranceSince;
|
|
235
|
+
/**
|
|
236
|
+
* Level-triggered spread recheck (see TIMING.SPREAD_STALE_RECHECK_MS).
|
|
237
|
+
* Armed by the out-of-spread watchdog while the grid is out of tolerance,
|
|
238
|
+
* since the base maintenance cadence can be hours.
|
|
239
|
+
*/
|
|
240
|
+
_spreadRecheckTimer;
|
|
241
|
+
_spreadRecheckInFlight;
|
|
242
|
+
/** "placed but did not reduce the violation" was already reported for this run. */
|
|
243
|
+
_outOfSpreadNoProgressWarned;
|
|
230
244
|
_postRecoveryRebalanceDefers;
|
|
231
245
|
_skipEmptyReadConfirmDelay;
|
|
232
246
|
_spreadStallArmed;
|
|
@@ -292,6 +306,9 @@ class DEXBot {
|
|
|
292
306
|
this._outOfSpreadStaleWarned = false;
|
|
293
307
|
this._outOfSpreadTransientCount = 0;
|
|
294
308
|
this._outOfSpreadToleranceSince = 0;
|
|
309
|
+
this._spreadRecheckTimer = null;
|
|
310
|
+
this._spreadRecheckInFlight = false;
|
|
311
|
+
this._outOfSpreadNoProgressWarned = false;
|
|
295
312
|
this.logPrefix = options.logPrefix || '';
|
|
296
313
|
this._credentialDaemonWatchdogInterval = null;
|
|
297
314
|
this._credentialDaemonDown = false;
|
|
@@ -610,7 +627,36 @@ class DEXBot {
|
|
|
610
627
|
_buildOrphanFillFallbackKey(fill) {
|
|
611
628
|
return DexbotFillRuntime.buildOrphanFillFallbackKey(this, fill);
|
|
612
629
|
}
|
|
630
|
+
/**
|
|
631
|
+
* Claim a fill key for this cycle, or report that it was already handled.
|
|
632
|
+
*
|
|
633
|
+
* Two authorities, in this order:
|
|
634
|
+
* 1. **The processed-fill store** — booked keys. Checked first and without
|
|
635
|
+
* any clock, because "already credited" does not expire. Before P2 a
|
|
636
|
+
* booked key older than the lease window was accepted again here and
|
|
637
|
+
* only refused later by accounting, so the replay still paid for queue
|
|
638
|
+
* churn and a FILL DETECTED log.
|
|
639
|
+
* 2. **The claim lease** — fills accepted but not yet booked (deferral,
|
|
640
|
+
* crash). Its TTL (`FILL_DEDUPE_WINDOW_MS`) is a lease, not the dedupe
|
|
641
|
+
* authority; the store refuses a booked key while retained, at any age.
|
|
642
|
+
*
|
|
643
|
+
* Claims are released on every path that does not book (requeue, orphan
|
|
644
|
+
* adoption, stale-totals deferral), so a held claim means "in flight".
|
|
645
|
+
*
|
|
646
|
+
* @param {string} fillKey - Chain fill identity (`orderId:blockNum:historyId`)
|
|
647
|
+
* @param {Set<string>} processedFillKeys - Per-cycle key set
|
|
648
|
+
* @param {string} [label] - Log prefix
|
|
649
|
+
* @param {string} [orderId] - Chain order id, for the log only
|
|
650
|
+
* @returns {boolean} true when this cycle may process the fill
|
|
651
|
+
*/
|
|
613
652
|
_isNewFillKey(fillKey, processedFillKeys, label = '', orderId = '') {
|
|
653
|
+
if (isFillBooked(this._processedFillStore, fillKey)) {
|
|
654
|
+
if (label) {
|
|
655
|
+
const idSuffix = orderId ? ` for ${orderId}` : '';
|
|
656
|
+
this.manager?.logger?.log?.(`${label} Skipping fill${idSuffix}: ${fillKey} is already booked`, 'debug');
|
|
657
|
+
}
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
614
660
|
const now = Date.now();
|
|
615
661
|
if (this._recentlyQueuedFills.has(fillKey)) {
|
|
616
662
|
const lastProcessed = this._recentlyQueuedFills.get(fillKey) ?? 0;
|
|
@@ -1027,6 +1073,11 @@ class DEXBot {
|
|
|
1027
1073
|
*/
|
|
1028
1074
|
async _executeBatchIfNeeded(rebalanceResult, contextLabel = 'rebalance') {
|
|
1029
1075
|
if (!hasExecutableActions(rebalanceResult)) {
|
|
1076
|
+
// A planned result that is not executed owns the plan region
|
|
1077
|
+
// (manager.performSafeRebalance acquires it before planning). No
|
|
1078
|
+
// COW body will adopt it on this path, so release it here or the
|
|
1079
|
+
// region parks every other placement path until the watchdog.
|
|
1080
|
+
releasePlanLease(rebalanceResult);
|
|
1030
1081
|
this.manager?.logger?.log?.(`[COW] No actions needed for ${contextLabel}`, 'debug');
|
|
1031
1082
|
// Structural-resync request from the plan path (unrecoverable
|
|
1032
1083
|
// boundary, rail-edge target, over-distance rotations): the plan
|
|
@@ -1487,10 +1538,23 @@ class DEXBot {
|
|
|
1487
1538
|
const rb = rebalanceResult;
|
|
1488
1539
|
if (!rb || !rb.workingGrid) {
|
|
1489
1540
|
const reason = 'NON_COW_PAYLOAD';
|
|
1541
|
+
// Never strand a plan region on a payload the executor cannot use.
|
|
1542
|
+
releasePlanLease(rebalanceResult);
|
|
1490
1543
|
this.manager?.logger?.log?.(`[COW] Rejected non-COW batch payload. Use updateOrdersOnChainPlan() for plan inputs.`, 'error');
|
|
1491
1544
|
return { executed: false, aborted: true, reason };
|
|
1492
1545
|
}
|
|
1493
|
-
|
|
1546
|
+
// Adopt the plan region carried by the result and hand it to the COW
|
|
1547
|
+
// body, which releases it on every exit (commit, refusal, throw).
|
|
1548
|
+
// Taking it here means a throw before the body adopts can be released
|
|
1549
|
+
// in the catch instead of parking the region (release is idempotent).
|
|
1550
|
+
const planLease = takePlanLease(rb);
|
|
1551
|
+
try {
|
|
1552
|
+
return await this._updateOrdersOnChainBatchCOW(rb, planLease ? { placementLease: planLease } : {});
|
|
1553
|
+
}
|
|
1554
|
+
catch (err) {
|
|
1555
|
+
planLease?.release();
|
|
1556
|
+
throw err;
|
|
1557
|
+
}
|
|
1494
1558
|
}
|
|
1495
1559
|
/**
|
|
1496
1560
|
* Converts simple plan payloads (place/update/rotate/cancel) into a COW batch.
|
|
@@ -1589,6 +1653,16 @@ class DEXBot {
|
|
|
1589
1653
|
* @private
|
|
1590
1654
|
*/
|
|
1591
1655
|
async _runStartupSequence() {
|
|
1656
|
+
// SINGLE-CONTROLLER LOCK: refuse to run when another live process
|
|
1657
|
+
// already controls this account (the double-controller incident of
|
|
1658
|
+
// 2026-11-14 had a `dexbot pm2` bot and a `dexbot start` bot trading
|
|
1659
|
+
// the same account for ~36h). Must wrap EVERY placement-producing
|
|
1660
|
+
// startup path (grid and credit-only alike), so it sits here before
|
|
1661
|
+
// the dispatch below. Released in _shutdownImpl; a crashed predecessor
|
|
1662
|
+
// is detected via /proc liveness and its lock taken over.
|
|
1663
|
+
if (this.account && !this._instanceLock) {
|
|
1664
|
+
this._instanceLock = acquireBotInstanceLock(String(this.account), String(this.config?.botKey || this.config?.name || ''), { log: (msg, level) => this._log(msg, level === 'warn' ? 'warn' : 'info') });
|
|
1665
|
+
}
|
|
1592
1666
|
try {
|
|
1593
1667
|
if (this.config.creditOnly) {
|
|
1594
1668
|
await this._runCreditOnlyStartup();
|
|
@@ -1980,6 +2054,10 @@ class DEXBot {
|
|
|
1980
2054
|
clearTimeout(this._maintenanceIdleTimer);
|
|
1981
2055
|
this._maintenanceIdleTimer = null;
|
|
1982
2056
|
}
|
|
2057
|
+
if (this._spreadRecheckTimer) {
|
|
2058
|
+
clearTimeout(this._spreadRecheckTimer);
|
|
2059
|
+
this._spreadRecheckTimer = null;
|
|
2060
|
+
}
|
|
1983
2061
|
if (this._credentialRecoveryDeferredTimer) {
|
|
1984
2062
|
clearTimeout(this._credentialRecoveryDeferredTimer);
|
|
1985
2063
|
this._credentialRecoveryDeferredTimer = null;
|
|
@@ -2145,6 +2223,13 @@ class DEXBot {
|
|
|
2145
2223
|
`lockContentions=${metrics.lockContentionEvents}, maxQueueDepth=${metrics.maxQueueDepth}, ` +
|
|
2146
2224
|
`heldChainOrders=${metrics.heldChainOrders ?? 0}, blockingChainOrders=${metrics.blockingChainOrders ?? 0}` +
|
|
2147
2225
|
refusalSuffix);
|
|
2226
|
+
// SINGLE-CONTROLLER LOCK: released last so a restart twin cannot slip
|
|
2227
|
+
// in while this instance still has pending broadcasts or persistence.
|
|
2228
|
+
if (this._instanceLock) {
|
|
2229
|
+
const lock = this._instanceLock;
|
|
2230
|
+
this._instanceLock = null;
|
|
2231
|
+
lock.release();
|
|
2232
|
+
}
|
|
2148
2233
|
await this.manager?.logger?.flush?.();
|
|
2149
2234
|
}
|
|
2150
2235
|
}
|
|
@@ -446,7 +446,8 @@ declare function isSpreadCorrectionBypassAllowed(bot: BotLike): boolean;
|
|
|
446
446
|
declare function refreshLastFillPivotFromQueue(bot: BotLike): boolean;
|
|
447
447
|
/**
|
|
448
448
|
* Build COW actions array from a simple plan object.
|
|
449
|
-
* @param {import('./dexbot_class.js').DEXBot}
|
|
449
|
+
* @param {import('./dexbot_class.js').DEXBot} _bot - Bot whose manager supplies the frozen
|
|
450
|
+
* boundary/gap geometry the plan stamp carries (plan values win when set).
|
|
450
451
|
* @param {Object|Array} plan
|
|
451
452
|
* @returns {Array}
|
|
452
453
|
*/
|
|
@@ -554,6 +555,35 @@ export declare function deriveRailBoundary(manager: OrderManagerLike, opContexts
|
|
|
554
555
|
*/
|
|
555
556
|
declare function trackBoundaryHold(manager: OrderManagerLike, held: boolean, keptBoundary: unknown, plannedBoundary: unknown, heldSlotIds: string[]): number;
|
|
556
557
|
declare function updateOrdersOnChainBatchCOW(bot: BotLike, cowResult: CowResult, options?: JsonObj): Promise<JsonObj>;
|
|
558
|
+
/**
|
|
559
|
+
* Converge master with the chain after a COW commit was refused (master moved
|
|
560
|
+
* during broadcast) or an uncertain broadcast was poll-confirmed.
|
|
561
|
+
*
|
|
562
|
+
* Preferred path (when the broadcast result is available): re-read the exact
|
|
563
|
+
* on-chain orders BY ID — every id master already tracks plus the batch's
|
|
564
|
+
* fresh CREATE ids. get_objects returns the complete, authoritative set
|
|
565
|
+
* regardless of account size, so syncFromOpenOrders converges master instead
|
|
566
|
+
* of dropping the freshest creates (which previously left permanent orphans
|
|
567
|
+
* and tripped the fund invariant). This is fully immune to the
|
|
568
|
+
* get_full_accounts window truncation that broke the old window-read path.
|
|
569
|
+
*
|
|
570
|
+
* Fallback (no broadcast result, e.g. uncertain paths, or by-id read
|
|
571
|
+
* unavailable): window read. A truncated read is ambiguous — only the
|
|
572
|
+
* freshest orders are dropped — so it MUST NOT drive adoption; we return false
|
|
573
|
+
* and let the caller keep pending-broadcast protection + structural resync.
|
|
574
|
+
*
|
|
575
|
+
* @param {unknown} bot
|
|
576
|
+
* @param {unknown} chainOrders - chain_orders module (has readOpenOrdersWithMetaSafe + batchReadOrders)
|
|
577
|
+
* @param {string} logPrefix
|
|
578
|
+
* @param {Object} [opts]
|
|
579
|
+
* @param {unknown} [opts.placedResults] - broadcast result carrying operation_results
|
|
580
|
+
* @param {unknown[]} [opts.placedContexts] - opContexts aligned with operation_results
|
|
581
|
+
* @param {string[]} [opts.polledCreateIds] - fresh CREATE chain ids confirmed by
|
|
582
|
+
* the uncertain-broadcast poll (no broadcast result exists on that path);
|
|
583
|
+
* routes the poll call sites through the by-id path with its lagging-create
|
|
584
|
+
* retry instead of the unguarded window fallback
|
|
585
|
+
* @returns {Promise<boolean>} true if master was adopted from the chain
|
|
586
|
+
*/
|
|
557
587
|
declare function adoptPlacedBatchFromChain(bot: BotLike, chainOrders: typeof chainOrdersModule, logPrefix: string, opts?: JsonObj): Promise<boolean>;
|
|
558
588
|
/**
|
|
559
589
|
* Resolve the LAST-FILL guard pivot to an ON-GRID price.
|