dexbot 1.7.2 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/grid_correction_check.d.ts +60 -1
- package/dist/analysis/grid_correction_check.js +268 -18
- package/dist/analysis/pivot_timeline.d.ts +78 -0
- package/dist/analysis/pivot_timeline.js +360 -0
- package/dist/market_adapter/utils/file_lock.js +16 -4
- package/dist/modules/account_bots.d.ts +6 -0
- package/dist/modules/bot_defaults.js +6 -1
- package/dist/modules/constants.d.ts +12 -0
- package/dist/modules/constants.js +66 -0
- package/dist/modules/dexbot_class.d.ts +17 -0
- package/dist/modules/dexbot_class.js +56 -1
- package/dist/modules/dexbot_cow_runtime.js +231 -19
- package/dist/modules/dexbot_maintenance_runtime.d.ts +31 -1
- package/dist/modules/dexbot_maintenance_runtime.js +164 -11
- package/dist/modules/dexbot_state_recovery.js +15 -1
- package/dist/modules/launcher/instance_lock.d.ts +27 -0
- package/dist/modules/launcher/instance_lock.js +131 -0
- package/dist/modules/order/grid.d.ts +19 -0
- package/dist/modules/order/grid.js +40 -14
- package/dist/modules/order/index.d.ts +15 -1
- package/dist/modules/order/manager.d.ts +21 -0
- package/dist/modules/order/manager.js +119 -12
- package/dist/modules/order/placement_lock.d.ts +30 -0
- package/dist/modules/order/placement_lock.js +48 -0
- package/dist/modules/order/utils/order.d.ts +39 -1
- package/dist/modules/order/utils/order.js +73 -2
- package/dist/modules/order/utils/system.d.ts +112 -3
- package/dist/modules/order/utils/system.js +368 -6
- package/dist/modules/paths.d.ts +1 -0
- package/dist/modules/paths.js +1 -0
- package/dist/modules/timing_policy.js +2 -0
- package/dist/modules/types.d.ts +27 -0
- package/dist/pm2.js +126 -0
- package/dist/unlock.js +26 -1
- package/docs/DEXBOT_COMPARISON.md +5 -5
- package/docs/EVOLUTION.md +10 -9
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +155 -2
- package/docs/README.md +1 -1
- package/modules/README.md +1 -0
- package/package.json +2 -1
|
@@ -82,12 +82,14 @@ import { PATHS } from './paths.js';
|
|
|
82
82
|
import { path } from './path_api.js';
|
|
83
83
|
import * as Format from './order/format.js';
|
|
84
84
|
import cowRuntime from './dexbot_cow_runtime.js';
|
|
85
|
+
import { takePlanLease, releasePlanLease } from './order/placement_lock.js';
|
|
85
86
|
import { ProcessedFillStore } from './order/processed_fill_store.js';
|
|
86
87
|
import { isFillBooked } from './order/fill_ledger.js';
|
|
87
88
|
import { TIMING, DAEMON_CODES, FILL_PROCESSING, } from './constants.js';
|
|
88
89
|
import { normalizeBotEntry } from './bot_settings.js';
|
|
89
90
|
import { createDefaultBotMetrics } from './bot_defaults.js';
|
|
90
91
|
import { getErrorMessage, getErrorCode } from './utils/errors.js';
|
|
92
|
+
import { acquireBotInstanceLock } from './launcher/instance_lock.js';
|
|
91
93
|
const waitForConnected = lazyShim('./bitshares_client', 'waitForConnected');
|
|
92
94
|
const getKeyStore = lazyShim('./key_store', 'getKeyStore');
|
|
93
95
|
const hasExecutableActions = lazyShim('./order/utils/validate', 'hasExecutableActions');
|
|
@@ -225,9 +227,20 @@ class DEXBot {
|
|
|
225
227
|
_marketAdapterWatchdogInFlight;
|
|
226
228
|
_outOfSpreadSince;
|
|
227
229
|
_outOfSpreadStaleWarned;
|
|
230
|
+
/** Held single-controller lock for this bot's account (null when not started). */
|
|
231
|
+
_instanceLock;
|
|
228
232
|
_outOfSpreadTransientCount;
|
|
229
233
|
/** When the spread first read within tolerance; cleared on any out-of-tolerance cycle. */
|
|
230
234
|
_outOfSpreadToleranceSince;
|
|
235
|
+
/**
|
|
236
|
+
* Level-triggered spread recheck (see TIMING.SPREAD_STALE_RECHECK_MS).
|
|
237
|
+
* Armed by the out-of-spread watchdog while the grid is out of tolerance,
|
|
238
|
+
* since the base maintenance cadence can be hours.
|
|
239
|
+
*/
|
|
240
|
+
_spreadRecheckTimer;
|
|
241
|
+
_spreadRecheckInFlight;
|
|
242
|
+
/** "placed but did not reduce the violation" was already reported for this run. */
|
|
243
|
+
_outOfSpreadNoProgressWarned;
|
|
231
244
|
_postRecoveryRebalanceDefers;
|
|
232
245
|
_skipEmptyReadConfirmDelay;
|
|
233
246
|
_spreadStallArmed;
|
|
@@ -293,6 +306,9 @@ class DEXBot {
|
|
|
293
306
|
this._outOfSpreadStaleWarned = false;
|
|
294
307
|
this._outOfSpreadTransientCount = 0;
|
|
295
308
|
this._outOfSpreadToleranceSince = 0;
|
|
309
|
+
this._spreadRecheckTimer = null;
|
|
310
|
+
this._spreadRecheckInFlight = false;
|
|
311
|
+
this._outOfSpreadNoProgressWarned = false;
|
|
296
312
|
this.logPrefix = options.logPrefix || '';
|
|
297
313
|
this._credentialDaemonWatchdogInterval = null;
|
|
298
314
|
this._credentialDaemonDown = false;
|
|
@@ -1057,6 +1073,11 @@ class DEXBot {
|
|
|
1057
1073
|
*/
|
|
1058
1074
|
async _executeBatchIfNeeded(rebalanceResult, contextLabel = 'rebalance') {
|
|
1059
1075
|
if (!hasExecutableActions(rebalanceResult)) {
|
|
1076
|
+
// A planned result that is not executed owns the plan region
|
|
1077
|
+
// (manager.performSafeRebalance acquires it before planning). No
|
|
1078
|
+
// COW body will adopt it on this path, so release it here or the
|
|
1079
|
+
// region parks every other placement path until the watchdog.
|
|
1080
|
+
releasePlanLease(rebalanceResult);
|
|
1060
1081
|
this.manager?.logger?.log?.(`[COW] No actions needed for ${contextLabel}`, 'debug');
|
|
1061
1082
|
// Structural-resync request from the plan path (unrecoverable
|
|
1062
1083
|
// boundary, rail-edge target, over-distance rotations): the plan
|
|
@@ -1517,10 +1538,23 @@ class DEXBot {
|
|
|
1517
1538
|
const rb = rebalanceResult;
|
|
1518
1539
|
if (!rb || !rb.workingGrid) {
|
|
1519
1540
|
const reason = 'NON_COW_PAYLOAD';
|
|
1541
|
+
// Never strand a plan region on a payload the executor cannot use.
|
|
1542
|
+
releasePlanLease(rebalanceResult);
|
|
1520
1543
|
this.manager?.logger?.log?.(`[COW] Rejected non-COW batch payload. Use updateOrdersOnChainPlan() for plan inputs.`, 'error');
|
|
1521
1544
|
return { executed: false, aborted: true, reason };
|
|
1522
1545
|
}
|
|
1523
|
-
|
|
1546
|
+
// Adopt the plan region carried by the result and hand it to the COW
|
|
1547
|
+
// body, which releases it on every exit (commit, refusal, throw).
|
|
1548
|
+
// Taking it here means a throw before the body adopts can be released
|
|
1549
|
+
// in the catch instead of parking the region (release is idempotent).
|
|
1550
|
+
const planLease = takePlanLease(rb);
|
|
1551
|
+
try {
|
|
1552
|
+
return await this._updateOrdersOnChainBatchCOW(rb, planLease ? { placementLease: planLease } : {});
|
|
1553
|
+
}
|
|
1554
|
+
catch (err) {
|
|
1555
|
+
planLease?.release();
|
|
1556
|
+
throw err;
|
|
1557
|
+
}
|
|
1524
1558
|
}
|
|
1525
1559
|
/**
|
|
1526
1560
|
* Converts simple plan payloads (place/update/rotate/cancel) into a COW batch.
|
|
@@ -1619,6 +1653,16 @@ class DEXBot {
|
|
|
1619
1653
|
* @private
|
|
1620
1654
|
*/
|
|
1621
1655
|
async _runStartupSequence() {
|
|
1656
|
+
// SINGLE-CONTROLLER LOCK: refuse to run when another live process
|
|
1657
|
+
// already controls this account (the double-controller incident of
|
|
1658
|
+
// 2026-11-14 had a `dexbot pm2` bot and a `dexbot start` bot trading
|
|
1659
|
+
// the same account for ~36h). Must wrap EVERY placement-producing
|
|
1660
|
+
// startup path (grid and credit-only alike), so it sits here before
|
|
1661
|
+
// the dispatch below. Released in _shutdownImpl; a crashed predecessor
|
|
1662
|
+
// is detected via /proc liveness and its lock taken over.
|
|
1663
|
+
if (this.account && !this._instanceLock) {
|
|
1664
|
+
this._instanceLock = acquireBotInstanceLock(String(this.account), String(this.config?.botKey || this.config?.name || ''), { log: (msg, level) => this._log(msg, level === 'warn' ? 'warn' : 'info') });
|
|
1665
|
+
}
|
|
1622
1666
|
try {
|
|
1623
1667
|
if (this.config.creditOnly) {
|
|
1624
1668
|
await this._runCreditOnlyStartup();
|
|
@@ -2010,6 +2054,10 @@ class DEXBot {
|
|
|
2010
2054
|
clearTimeout(this._maintenanceIdleTimer);
|
|
2011
2055
|
this._maintenanceIdleTimer = null;
|
|
2012
2056
|
}
|
|
2057
|
+
if (this._spreadRecheckTimer) {
|
|
2058
|
+
clearTimeout(this._spreadRecheckTimer);
|
|
2059
|
+
this._spreadRecheckTimer = null;
|
|
2060
|
+
}
|
|
2013
2061
|
if (this._credentialRecoveryDeferredTimer) {
|
|
2014
2062
|
clearTimeout(this._credentialRecoveryDeferredTimer);
|
|
2015
2063
|
this._credentialRecoveryDeferredTimer = null;
|
|
@@ -2175,6 +2223,13 @@ class DEXBot {
|
|
|
2175
2223
|
`lockContentions=${metrics.lockContentionEvents}, maxQueueDepth=${metrics.maxQueueDepth}, ` +
|
|
2176
2224
|
`heldChainOrders=${metrics.heldChainOrders ?? 0}, blockingChainOrders=${metrics.blockingChainOrders ?? 0}` +
|
|
2177
2225
|
refusalSuffix);
|
|
2226
|
+
// SINGLE-CONTROLLER LOCK: released last so a restart twin cannot slip
|
|
2227
|
+
// in while this instance still has pending broadcasts or persistence.
|
|
2228
|
+
if (this._instanceLock) {
|
|
2229
|
+
const lock = this._instanceLock;
|
|
2230
|
+
this._instanceLock = null;
|
|
2231
|
+
lock.release();
|
|
2232
|
+
}
|
|
2178
2233
|
await this.manager?.logger?.flush?.();
|
|
2179
2234
|
}
|
|
2180
2235
|
}
|
|
@@ -14,14 +14,15 @@ import { BroadcastUncertainError as BroadcastUncertainErrorBinding } from './dex
|
|
|
14
14
|
const BroadcastUncertainError = BroadcastUncertainErrorBinding;
|
|
15
15
|
import * as orderUtils from './order/utils/order.js';
|
|
16
16
|
import { lazyShim } from './lazy_shim.js';
|
|
17
|
-
import { sleep, setLastFillPivot, fireStructuralResync, evaluateSpreadCorrectionReadiness, resolveLastFillGuardReference, resolveSpreadCrossingFloor, isLastFillGuardBlocked } from './order/utils/system.js';
|
|
18
|
-
const { buildCreateOrderArgs, buildCreateOpFingerprint, extractBatchOperationResults, formatUnmatchedChainOrder, convertToSpreadPlaceholder, toRailHolePlaceholder, buildOutsideInPairGroups, isOrderPlaced, chainOrderUnchangedFromCache, detectCrossedBookPlan, collectKnownOnChainOrderIds, } = orderUtils;
|
|
17
|
+
import { sleep, setLastFillPivot, fireStructuralResync, evaluateSpreadCorrectionReadiness, resolveLastFillGuardReference, resolveSpreadCrossingFloor, isLastFillGuardBlocked, evaluateLastFillGuard } from './order/utils/system.js';
|
|
18
|
+
const { buildCreateOrderArgs, buildCreateOpFingerprint, extractBatchOperationResults, formatUnmatchedChainOrder, convertToSpreadPlaceholder, toRailHolePlaceholder, buildOutsideInPairGroups, isOrderPlaced, chainOrderUnchangedFromCache, detectCrossedBookPlan, describeMarketCrossingActions, collectKnownOnChainOrderIds, } = orderUtils;
|
|
19
19
|
import * as validate from './order/utils/validate.js';
|
|
20
20
|
const { validateCreateTargetSlots, evaluateCommit, hasExecutableActions, stampGapEvacuationRotation } = validate;
|
|
21
21
|
import * as math from './order/utils/math.js';
|
|
22
22
|
const { validateOrderSize, findCrossedOrder, sameGridLevel, isEvacuationRotationAllowed, isEvacuationSizeStillValid, getSellStartIdx, getPrecisionByOrderType, isSlotIndexInGapBand, getAssetFeesSafe, blockchainToFloat, floatToBlockchainInt, quantizeFloat } = math;
|
|
23
23
|
import { parseSlotIndex } from './order/utils/slot.js';
|
|
24
24
|
import { resolveTradedSide } from './order/fill_ledger.js';
|
|
25
|
+
import { takePlanLease } from './order/placement_lock.js';
|
|
25
26
|
/**
|
|
26
27
|
* Re-verify a B-stamped gap-evacuation rotation against the LIVE committed
|
|
27
28
|
* geometry (boundaryIdx/_gapSlots at execution time). The stamp freezes
|
|
@@ -237,6 +238,12 @@ import { unknownCowOutcome } from './outcome.js';
|
|
|
237
238
|
* never heals on a quiet market.
|
|
238
239
|
*/
|
|
239
240
|
const scheduleGapEvacRepair = lazyShim('./dexbot_state_recovery', 'scheduleGapEvacRepair');
|
|
241
|
+
/**
|
|
242
|
+
* Post-repair re-plan (lazy, same reason as the sweep above). A targeted repair
|
|
243
|
+
* virtualises the dead slot but cannot complete the rotation that slot's batch
|
|
244
|
+
* was carrying, so the grid needs one fresh plan to re-place the rail.
|
|
245
|
+
*/
|
|
246
|
+
const schedulePostRecoveryRebalance = lazyShim('./dexbot_state_recovery', 'schedulePostRecoveryRebalance');
|
|
240
247
|
// Maximum number of times the pre-broadcast staleness guard may re-plan the
|
|
241
248
|
// batch from a fresh master before proceeding anyway. Bounded so a master
|
|
242
249
|
// grid that keeps mutating (fill bursts, sync loops) can never livelock the
|
|
@@ -1500,6 +1507,15 @@ async function executeChunkedWithRetryOnUncertain(bot, operations, opContexts) {
|
|
|
1500
1507
|
// mode already stops at the first failed group; the chunked wrapper
|
|
1501
1508
|
// should match — continuing would only burn rejected broadcasts.
|
|
1502
1509
|
abortedChunkCount = chunks.length - idx - 1;
|
|
1510
|
+
if (bot._metrics && abortedChunkCount > 0) {
|
|
1511
|
+
// Planned operations thrown away by this abort (the remaining
|
|
1512
|
+
// chunks are never sent). The plan's work is lost; the metric
|
|
1513
|
+
// makes the cost of a single stale target visible.
|
|
1514
|
+
let abortedOps = 0;
|
|
1515
|
+
for (let k = idx + 1; k < chunks.length; k++)
|
|
1516
|
+
abortedOps += chunks[k].operations.length;
|
|
1517
|
+
bot._metrics.chunkAbortedOpCount = (bot._metrics.chunkAbortedOpCount || 0) + abortedOps;
|
|
1518
|
+
}
|
|
1503
1519
|
bot.manager.logger.log(`[COW] Chunk ${idx + 1}/${chunks.length} broadcast failed definitively (${getErrorMessage(err)}); ` +
|
|
1504
1520
|
`aborting the remaining ${abortedChunkCount} chunk(s) — the rejected chunk landed nothing.`, 'error');
|
|
1505
1521
|
break;
|
|
@@ -2395,10 +2411,28 @@ function buildCowResultFromPlan(bot, plan) {
|
|
|
2395
2411
|
});
|
|
2396
2412
|
}
|
|
2397
2413
|
else if (action.type === COW_ACTIONS.UPDATE) {
|
|
2414
|
+
// A ROTATION is TWO effects in one op: the SOURCE becomes a hole at
|
|
2415
|
+
// ITS OWN ladder level and the DESTINATION takes the order at the
|
|
2416
|
+
// destination level. The source is not part of the UPDATE payload —
|
|
2417
|
+
// falling through to the in-place branch below copied `action.order`
|
|
2418
|
+
// (the DESTINATION's price and size) onto the vacated slot, so two
|
|
2419
|
+
// slots ended up on one level. Live: 6 duplicated levels in a
|
|
2420
|
+
// persisted grid, a non-monotonic ladder, a shifted boundary index
|
|
2421
|
+
// and ~32k of double-reserved virtual intent. Apply and leave; the
|
|
2422
|
+
// in-place branch is for a partial-fill size update (same slot).
|
|
2398
2423
|
if (action.newGridId && action.newGridId !== action.id) {
|
|
2399
2424
|
const current = workingGrid.get(action.id);
|
|
2400
2425
|
if (current) {
|
|
2401
|
-
|
|
2426
|
+
// GRID-PRICE-INVARIANT at the writer: a hole keeps the
|
|
2427
|
+
// level of the slot it vacated. Asserted (not assumed) so a
|
|
2428
|
+
// future placeholder change cannot re-open the class — the
|
|
2429
|
+
// emission-site checks never see this slot because no op
|
|
2430
|
+
// targets it.
|
|
2431
|
+
const sourceLevel = Number(current.price);
|
|
2432
|
+
const hole = toWorkingHole(current);
|
|
2433
|
+
workingGrid.set(action.id, Number.isFinite(sourceLevel) && sourceLevel > 0
|
|
2434
|
+
? { ...hole, price: sourceLevel }
|
|
2435
|
+
: hole);
|
|
2402
2436
|
}
|
|
2403
2437
|
const targetId = action.newGridId;
|
|
2404
2438
|
const targetCurrent = workingGrid.get(targetId) || { id: targetId };
|
|
@@ -2417,6 +2451,7 @@ function buildCowResultFromPlan(bot, plan) {
|
|
|
2417
2451
|
state: ORDER_STATES.VIRTUAL,
|
|
2418
2452
|
orderId: null,
|
|
2419
2453
|
});
|
|
2454
|
+
continue;
|
|
2420
2455
|
}
|
|
2421
2456
|
const current = workingGrid.get(action.id);
|
|
2422
2457
|
if (!current)
|
|
@@ -3142,13 +3177,52 @@ async function replanStaleBatch(bot, cowResult, replanDepth, preBroadcastGuard,
|
|
|
3142
3177
|
result: { executed: false, aborted: true, reason: 'STRUCTURAL_REBUILD_DEFERRED', hadRotation: false }
|
|
3143
3178
|
};
|
|
3144
3179
|
}
|
|
3145
|
-
const
|
|
3146
|
-
|
|
3180
|
+
const hasFillContext = Array.isArray(cowResult.fills) && cowResult.fills.length > 0;
|
|
3181
|
+
const canReplan = replanDepth < STALE_PLAN_REPLAN_LIMIT && hasFillContext;
|
|
3147
3182
|
if (!canReplan) {
|
|
3183
|
+
if (!hasFillContext) {
|
|
3184
|
+
// NO FILL CONTEXT → DROP, DON'T SHIP. The master mutated while this
|
|
3185
|
+
// batch was being planned (dominant cause: a snapshot `grid-load`,
|
|
3186
|
+
// also surplus cancels / handle-fill batches) and there is no fill
|
|
3187
|
+
// set to re-plan from. The bounded "proceed anyway" policy exists
|
|
3188
|
+
// because an abort would drop a fill set — that rationale does not
|
|
3189
|
+
// apply here. Shipping it broadcast UPDATE/CREATE ops computed from
|
|
3190
|
+
// the pre-mutation slot/orderId map: observed as missing UPDATE
|
|
3191
|
+
// targets (repair), duplicate creates at the same level, refused
|
|
3192
|
+
// commits, fund drift → snapshot reject → another reload.
|
|
3193
|
+
// Dropping is level-triggered: `_deferredRebalanceAt` makes the
|
|
3194
|
+
// region-end hook (placement release below) schedule the no-fill
|
|
3195
|
+
// rebalance that re-derives from the fresh master.
|
|
3196
|
+
if (bot._metrics) {
|
|
3197
|
+
bot._metrics.stalePlanDroppedCount = (bot._metrics.stalePlanDroppedCount || 0) + 1;
|
|
3198
|
+
}
|
|
3199
|
+
bot.manager.logger.log(`[COW] Plan stale pre-broadcast (${preBroadcastGuard.reason}); no fill context to re-plan ` +
|
|
3200
|
+
`— dropping the batch instead of broadcasting stale slot state`, 'warn');
|
|
3201
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
3202
|
+
clearPendingBroadcastsForSlots(bot, cowResult.actions);
|
|
3203
|
+
if (bot.manager && typeof bot.manager === 'object') {
|
|
3204
|
+
bot.manager._deferredRebalanceAt = Date.now();
|
|
3205
|
+
}
|
|
3206
|
+
return {
|
|
3207
|
+
handled: true,
|
|
3208
|
+
result: {
|
|
3209
|
+
executed: false,
|
|
3210
|
+
aborted: true,
|
|
3211
|
+
deferred: true,
|
|
3212
|
+
droppedStalePlan: true,
|
|
3213
|
+
reason: 'STALE_PLAN_NO_FILL_CONTEXT',
|
|
3214
|
+
hadRotation: false
|
|
3215
|
+
}
|
|
3216
|
+
};
|
|
3217
|
+
}
|
|
3218
|
+
// Fill batch that is still stale at the re-plan limit: keep the
|
|
3219
|
+
// documented bounded policy (proceed + resync) — the fill set is real
|
|
3220
|
+
// and the post-broadcast commit guard + chain adoption close divergence.
|
|
3221
|
+
if (bot._metrics) {
|
|
3222
|
+
bot._metrics.stalePlanProceedCount = (bot._metrics.stalePlanProceedCount || 0) + 1;
|
|
3223
|
+
}
|
|
3148
3224
|
bot.manager.logger.log(`[COW] Plan stale pre-broadcast (${preBroadcastGuard.reason}); ` +
|
|
3149
|
-
(
|
|
3150
|
-
? 'still stale after re-plan — proceeding with plan (commit guard + chain adoption close divergence)'
|
|
3151
|
-
: 'no fill context for re-plan — proceeding with plan (commit guard + chain adoption close divergence)'), 'warn');
|
|
3225
|
+
'still stale after re-plan — proceeding with plan (commit guard + chain adoption close divergence)', 'warn');
|
|
3152
3226
|
await requestStructuralResync(bot, 'plan stale pre-broadcast (proceeding with plan)', { reason: preBroadcastGuard.reason });
|
|
3153
3227
|
return { handled: false };
|
|
3154
3228
|
}
|
|
@@ -3625,8 +3699,69 @@ async function runPreBroadcastGuards(bot, cowResult) {
|
|
|
3625
3699
|
if (missing.length > 0) {
|
|
3626
3700
|
bot.manager.logger.log(`[COW] Rejecting UPDATE batch pre-broadcast: ${missing.length} target(s) missing on chain ` +
|
|
3627
3701
|
`(${missing.slice(0, 8).join(', ')}${missing.length > 8 ? ', …' : ''}); ` +
|
|
3628
|
-
`grid/chain existence divergence
|
|
3702
|
+
`grid/chain existence divergence`, 'error');
|
|
3629
3703
|
popPushedWorkingGrid(bot, cowResult);
|
|
3704
|
+
// REPAIR THE SLOT, NOT THE WHOLE GRID. Both reads above agreed
|
|
3705
|
+
// these ids are absent, so each one is a slot whose chain order
|
|
3706
|
+
// is gone (a missed fill or a phantom left by an earlier
|
|
3707
|
+
// reload). The old response — a structural resync, which
|
|
3708
|
+
// reloads the on-disk snapshot — could only re-read the same
|
|
3709
|
+
// stale slot from disk and plan the same UPDATE again: the
|
|
3710
|
+
// refused batch re-armed itself forever, and while that slot
|
|
3711
|
+
// sat "placed" inside the gap band every boundary slide
|
|
3712
|
+
// (hence every sell-side promotion) stayed refused, so the
|
|
3713
|
+
// spread never closed. `_targetedOrderRepair` is the existing
|
|
3714
|
+
// single-slot repair (chain read -> placeholder when absent),
|
|
3715
|
+
// already used by the size-drift path; it releases the dead
|
|
3716
|
+
// commitment and leaves the level as a plain hole.
|
|
3717
|
+
let repaired = false;
|
|
3718
|
+
if (typeof bot._targetedOrderRepair === 'function') {
|
|
3719
|
+
try {
|
|
3720
|
+
repaired = (await bot._targetedOrderRepair(missing)) === true;
|
|
3721
|
+
}
|
|
3722
|
+
catch (repairErr) {
|
|
3723
|
+
bot.manager.logger.log(`[COW] Targeted repair of missing UPDATE target(s) failed (${getErrorMessage(repairErr)}); ` +
|
|
3724
|
+
`falling back to structural resync`, 'warn');
|
|
3725
|
+
repaired = false;
|
|
3726
|
+
}
|
|
3727
|
+
}
|
|
3728
|
+
if (repaired) {
|
|
3729
|
+
// A repair that leaves an id tracked did nothing (e.g. the
|
|
3730
|
+
// id belongs to no grid slot): that must escalate, not
|
|
3731
|
+
// silently drop a batch per cycle.
|
|
3732
|
+
const stillTracked = new Set();
|
|
3733
|
+
for (const order of bot.manager.orders.values()) {
|
|
3734
|
+
const trackedId = order?.orderId ? String(order.orderId) : '';
|
|
3735
|
+
if (trackedId && missing.includes(trackedId))
|
|
3736
|
+
stillTracked.add(trackedId);
|
|
3737
|
+
}
|
|
3738
|
+
if (stillTracked.size > 0) {
|
|
3739
|
+
bot.manager.logger.log(`[COW] Targeted repair left ${stillTracked.size} missing target(s) tracked ` +
|
|
3740
|
+
`(${[...stillTracked].slice(0, 8).join(', ')}); falling back to structural resync`, 'warn');
|
|
3741
|
+
repaired = false;
|
|
3742
|
+
}
|
|
3743
|
+
}
|
|
3744
|
+
if (repaired) {
|
|
3745
|
+
bot.manager.logger.log(`[COW] Repaired ${missing.length} missing UPDATE target(s) locally ` +
|
|
3746
|
+
`(${missing.slice(0, 8).join(', ')}${missing.length > 8 ? ', …' : ''}); ` +
|
|
3747
|
+
`batch dropped and a fresh plan scheduled — no snapshot reload`, 'warn');
|
|
3748
|
+
schedulePostRecoveryRebalance(bot, `missing update target repair (${missing.join(', ')})`);
|
|
3749
|
+
return {
|
|
3750
|
+
proceed: false,
|
|
3751
|
+
result: {
|
|
3752
|
+
executed: false,
|
|
3753
|
+
aborted: true,
|
|
3754
|
+
// Reason stays UPDATE_TARGET_MISSING so every
|
|
3755
|
+
// existing classifier/metric keeps its meaning;
|
|
3756
|
+
// `repairedMissingTargets` tells the re-plan path
|
|
3757
|
+
// that the grid was repaired rather than reloaded.
|
|
3758
|
+
reason: 'UPDATE_TARGET_MISSING',
|
|
3759
|
+
missingUpdateOrderIds: missing,
|
|
3760
|
+
repairedMissingTargets: true,
|
|
3761
|
+
hadRotation: false
|
|
3762
|
+
}
|
|
3763
|
+
};
|
|
3764
|
+
}
|
|
3630
3765
|
await requestStructuralResync(bot, 'pre-broadcast update target missing on chain', { missingUpdateOrderIds: missing });
|
|
3631
3766
|
return {
|
|
3632
3767
|
proceed: false,
|
|
@@ -3706,6 +3841,25 @@ async function runPreBroadcastGuards(bot, cowResult) {
|
|
|
3706
3841
|
}
|
|
3707
3842
|
};
|
|
3708
3843
|
}
|
|
3844
|
+
// MARKET-CROSSING OBSERVATION (retrospective §9.10/P4): actions priced on
|
|
3845
|
+
// the wrong side of the market floor are REPORTED, never dropped. A hard
|
|
3846
|
+
// veto here re-opens the counter-order deadlock P4 rejected — after a gap
|
|
3847
|
+
// the counter sits across the floor by construction, and the live cost of
|
|
3848
|
+
// the veto was a batch refused at identical prices that the next tick
|
|
3849
|
+
// accepted once the reference moved (86 minutes out of spread). The
|
|
3850
|
+
// crossed-book gate above still protects the batch internally (buy >= our
|
|
3851
|
+
// own sell), and the market prices a genuinely crossing op as a taker fill.
|
|
3852
|
+
try {
|
|
3853
|
+
const { floor, dropped } = describeMarketCrossingActions(bot.manager, actions);
|
|
3854
|
+
if (dropped.length > 0) {
|
|
3855
|
+
bot.manager.logger.log(`[COW-CROSS-GUARD] ${dropped.length}/${actions.length} action(s) priced across the market floor `
|
|
3856
|
+
+ `${floor != null ? Format.formatPrice6(floor) : 'n/a'} `
|
|
3857
|
+
+ `(${dropped.map((d) => d.reason).slice(0, 4).join('; ')}`
|
|
3858
|
+
+ `${dropped.length > 4 ? '; …' : ''}) — observation only, batch broadcast unchanged `
|
|
3859
|
+
+ `(a hard veto re-opens the counter-order deadlock; retrospective §9.10/P4)`, 'warn');
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
catch { /* the observation must never break a batch */ }
|
|
3709
3863
|
return { proceed: true, crossingCandidates, intraBatchCandidates };
|
|
3710
3864
|
}
|
|
3711
3865
|
function restoreTestPollIntervalSeam(bot, prev) {
|
|
@@ -3726,6 +3880,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult, options = {}) {
|
|
|
3726
3880
|
// branch and any re-plan recursion pick it up without changing the
|
|
3727
3881
|
// production call signature used by the runtime.
|
|
3728
3882
|
const seamPollIntervalMs = resolveSeamMsOrNull(options?.pollIntervalMs);
|
|
3883
|
+
let resultPlanLease = null;
|
|
3729
3884
|
const prevSeamPollIntervalMs = bot?._testPollIntervalMs;
|
|
3730
3885
|
if (seamPollIntervalMs != null) {
|
|
3731
3886
|
bot._testPollIntervalMs = seamPollIntervalMs;
|
|
@@ -3742,9 +3897,18 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult, options = {}) {
|
|
|
3742
3897
|
// recursion, throws) funnels through the body() finally below, so no exit
|
|
3743
3898
|
// can leak bot._testPollIntervalMs onto the bot.
|
|
3744
3899
|
try {
|
|
3745
|
-
|
|
3900
|
+
// Plan region carried by the result (manager.performSafeRebalance
|
|
3901
|
+
// acquires it before planning). Direct callers of this function (tests,
|
|
3902
|
+
// the re-plan shim) may hand it over here instead of as an option.
|
|
3903
|
+
resultPlanLease = takePlanLease(cowResult);
|
|
3904
|
+
return await updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seamPollIntervalMs ?? undefined, resultPlanLease ?? options?.placementLease ?? null);
|
|
3746
3905
|
}
|
|
3747
3906
|
finally {
|
|
3907
|
+
// Idempotent safety release: the body releases the inherited lease on
|
|
3908
|
+
// every one of its own exits; this covers a throw before the body's
|
|
3909
|
+
// try/finally takes ownership, so a direct caller cannot strand the
|
|
3910
|
+
// region on the result path.
|
|
3911
|
+
resultPlanLease?.release();
|
|
3748
3912
|
// Restore only when this frame actually wrote a seam. The entry
|
|
3749
3913
|
// single-flight await (inside body()) lets a second, seam-less batch
|
|
3750
3914
|
// run concurrently: an unguarded restore would delete the seam owned
|
|
@@ -4158,8 +4322,9 @@ async function updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seam
|
|
|
4158
4322
|
lastFillGuardPivotRefreshed = true;
|
|
4159
4323
|
if (check.blocked) {
|
|
4160
4324
|
lastFillGuardStats.skipped++;
|
|
4161
|
-
|
|
4162
|
-
|
|
4325
|
+
bot.manager.logger.log(check.refusedLevel
|
|
4326
|
+
? `[LAST-FILL-GUARD] Skipping ${order.type} CREATE for ${order.id} at ${Format.formatPrice6(createPrice)}: same-level re-fill of the just-filled level ${Format.formatPrice6(Number(check.pivot))} (its pivot-arm was refused; the level stays blocked for the marker TTL, §9.12/P2 amended); re-planned after market moves`
|
|
4327
|
+
: `[LAST-FILL-GUARD] Skipping ${order.type} CREATE for ${order.id} at ${Format.formatPrice6(createPrice)}: ${order.type === ORDER_TYPES.BUY ? 'above' : 'below'} last filled ${Format.formatPrice6(Number(check.pivot))} (direction rule: BUY must be below / SELL above the last fill); re-planned after market moves`, 'debug');
|
|
4163
4328
|
if (order.id)
|
|
4164
4329
|
skippedCreateSlotIds.add(order.id);
|
|
4165
4330
|
continue;
|
|
@@ -4503,8 +4668,9 @@ async function updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seam
|
|
|
4503
4668
|
skippedUpdateSlotIds.add(action.id);
|
|
4504
4669
|
if (action.newGridId)
|
|
4505
4670
|
skippedUpdateSlotIds.add(action.newGridId);
|
|
4506
|
-
|
|
4507
|
-
|
|
4671
|
+
bot.manager.logger.log(check.refusedLevel
|
|
4672
|
+
? `[LAST-FILL-GUARD] Skipping ${orderType} UPDATE for ${action.id} -> ${action.newGridId} at ${Format.formatPrice6(newPrice)}: same-level re-fill of the just-filled level ${Format.formatPrice6(Number(check.pivot))} (its pivot-arm was refused; the level stays blocked for the marker TTL, §9.12/P2 amended)`
|
|
4673
|
+
: `[LAST-FILL-GUARD] Skipping ${orderType} UPDATE for ${action.id} -> ${action.newGridId} at ${Format.formatPrice6(newPrice)}: ${orderType === ORDER_TYPES.BUY ? 'above' : 'below'} last filled ${Format.formatPrice6(Number(check.pivot))} (direction rule: BUY must be below / SELL above the last fill)`, 'debug');
|
|
4508
4674
|
continue;
|
|
4509
4675
|
}
|
|
4510
4676
|
lastFillGuardStats.passed++;
|
|
@@ -4556,7 +4722,29 @@ async function updateOrdersOnChainBatchCOWBody(bot, cowResult, replanDepth, seam
|
|
|
4556
4722
|
const plannedNewSize = plannedUpdateSize(action);
|
|
4557
4723
|
const newSize = clampPostFillUpdateSize(bot, masterOrder, plannedNewSize, action);
|
|
4558
4724
|
if (newSize !== plannedNewSize) {
|
|
4725
|
+
// The clamp only fires when the plan wanted to GROW a
|
|
4726
|
+
// PARTIAL slot; the target is the booked remainder,
|
|
4727
|
+
// i.e. the size already on chain. Building the op would
|
|
4728
|
+
// re-read the order from chain and return null (observed:
|
|
4729
|
+
// 1 136/1 137 "no blockchain delta" skips were this
|
|
4730
|
+
// clamp), so skip it here like the rotation path does
|
|
4731
|
+
// (see the POST-FILL GROWTH GUARD above) — same rule, one
|
|
4732
|
+
// predicate, no wasted chain read. The slot still joins
|
|
4733
|
+
// clampedUpdateSlotIds so the commit re-syncs it from
|
|
4734
|
+
// master (the working grid holds the refused larger size).
|
|
4559
4735
|
clampedUpdateSlotIds.add(action.id);
|
|
4736
|
+
skippedUpdateCount++;
|
|
4737
|
+
if (action.id)
|
|
4738
|
+
skippedUpdateSlotIds.add(action.id);
|
|
4739
|
+
if (action.newGridId)
|
|
4740
|
+
skippedUpdateSlotIds.add(action.newGridId);
|
|
4741
|
+
if (bot._metrics) {
|
|
4742
|
+
bot._metrics.clampedNoopUpdateCount = (bot._metrics.clampedNoopUpdateCount || 0) + 1;
|
|
4743
|
+
}
|
|
4744
|
+
bot.manager.logger.log(`[COW] Skipping size update ${action.id} (${action.orderId}): ` +
|
|
4745
|
+
`PARTIAL slot at its booked remaining ${Format.formatAmount(newSize)} — ` +
|
|
4746
|
+
`growth in place refused, no chain change`, 'debug');
|
|
4747
|
+
continue;
|
|
4560
4748
|
}
|
|
4561
4749
|
const orderType = action.order?.type || masterOrder?.type;
|
|
4562
4750
|
const cachedRawOnChain = masterOrder?.rawOnChain || action.order?.rawOnChain || null;
|
|
@@ -5770,7 +5958,18 @@ function runLastFillGuardCheck(bot, price, type, stats, skipRefresh = false) {
|
|
|
5770
5958
|
catch { /* logging is best-effort */ }
|
|
5771
5959
|
stats.pivotOffGrid = (stats.pivotOffGrid || 0) + 1;
|
|
5772
5960
|
}
|
|
5773
|
-
const check =
|
|
5961
|
+
const check = evaluateLastFillGuard(bot.manager, price, type, onGrid.price, lastType);
|
|
5962
|
+
// Geometry-territory exemption (see isLastFillBandSellExempt): a SELL that
|
|
5963
|
+
// closes a band the fill anchor itself derived is not a "lower sell" the
|
|
5964
|
+
// guard is meant to refuse — refusing it vetoes 100% of the counter-orders.
|
|
5965
|
+
if (check.bandExempt) {
|
|
5966
|
+
try {
|
|
5967
|
+
bot.manager?.logger?.log?.(`[LAST-FILL-GUARD] Allowing SELL ${Format.formatPrice6(price)} against pivot `
|
|
5968
|
+
+ `${Format.formatPrice6(Number(onGrid.price))}: inside the gap band the fill anchor derives `
|
|
5969
|
+
+ `(boundary ${bot.manager?.boundaryIdx}/gap ${bot.manager?._gapSlots}) — geometry is not behind the fill`, 'debug');
|
|
5970
|
+
}
|
|
5971
|
+
catch { /* logging is best-effort */ }
|
|
5972
|
+
}
|
|
5774
5973
|
stats.checked++;
|
|
5775
5974
|
return { check, refreshed };
|
|
5776
5975
|
}
|
|
@@ -6053,12 +6252,20 @@ function runFinalPivotGate(bot, operations, opContexts, opts = {}) {
|
|
|
6053
6252
|
continue;
|
|
6054
6253
|
if (type !== ORDER_TYPES.BUY && type !== ORDER_TYPES.SELL)
|
|
6055
6254
|
continue;
|
|
6056
|
-
const check =
|
|
6255
|
+
const check = evaluateLastFillGuard(bot.manager, price, type, onGrid.price, liveType);
|
|
6057
6256
|
if (stats)
|
|
6058
6257
|
stats.checked = (Number(stats.checked) || 0) + 1;
|
|
6059
6258
|
if (!check.blocked) {
|
|
6060
6259
|
if (stats)
|
|
6061
6260
|
stats.passed = (Number(stats.passed) || 0) + 1;
|
|
6261
|
+
if (check.bandExempt) {
|
|
6262
|
+
try {
|
|
6263
|
+
bot.manager?.logger?.log?.(`[LAST-FILL-GUARD] Final gate keeping ${type} ${ctx.kind} for ${slotId ?? 'unknown'} at `
|
|
6264
|
+
+ `${Format.formatPrice6(price)}: inside the gap band the fill anchor derives `
|
|
6265
|
+
+ `(boundary ${bot.manager?.boundaryIdx}/gap ${bot.manager?._gapSlots})`, 'debug');
|
|
6266
|
+
}
|
|
6267
|
+
catch { /* logging is best-effort */ }
|
|
6268
|
+
}
|
|
6062
6269
|
continue;
|
|
6063
6270
|
}
|
|
6064
6271
|
// Blocked: drop the op + context, restore the slot below.
|
|
@@ -6068,9 +6275,14 @@ function runFinalPivotGate(bot, operations, opContexts, opts = {}) {
|
|
|
6068
6275
|
const dir = type === ORDER_TYPES.BUY ? 'above' : 'below';
|
|
6069
6276
|
dropped.push({ index: i, kind: ctx.kind, slotId, price, type });
|
|
6070
6277
|
try {
|
|
6071
|
-
bot.manager?.logger?.log?.(
|
|
6072
|
-
|
|
6073
|
-
|
|
6278
|
+
bot.manager?.logger?.log?.(check.refusedLevel
|
|
6279
|
+
? `[LAST-FILL-GUARD] Final gate dropping ${type} ${ctx.kind} for ${slotId ?? 'unknown'} at ` +
|
|
6280
|
+
`${Format.formatPrice6(price)}: same-level re-fill of the just-filled level ` +
|
|
6281
|
+
`${check.pivot != null ? Format.formatPrice6(Number(check.pivot)) : 'n/a'} ` +
|
|
6282
|
+
`(its pivot-arm was refused; the level stays blocked for the marker TTL, §9.12/P2 amended)`
|
|
6283
|
+
: `[LAST-FILL-GUARD] Final gate dropping ${type} ${ctx.kind} for ${slotId ?? 'unknown'} at ` +
|
|
6284
|
+
`${Format.formatPrice6(price)}: ${dir} last filled ${check.pivot != null ? Format.formatPrice6(Number(check.pivot)) : 'n/a'} ` +
|
|
6285
|
+
`(direction rule: BUY must be below / SELL above the last fill); re-planned after market moves`, 'warn');
|
|
6074
6286
|
}
|
|
6075
6287
|
catch { /* logging is best-effort */ }
|
|
6076
6288
|
}
|
|
@@ -324,6 +324,27 @@ declare function checkGridLockHoldDuration(bot: BotLike): number;
|
|
|
324
324
|
* @returns {Promise<void>}
|
|
325
325
|
*/
|
|
326
326
|
declare function executeMaintenanceLogic(bot: BotLike, context: string): Promise<void>;
|
|
327
|
+
/**
|
|
328
|
+
* Arm a one-shot, level-triggered spread recheck (TIMING.SPREAD_STALE_RECHECK_MS).
|
|
329
|
+
*
|
|
330
|
+
* Retrospective R5.5: "Make deferred work level-triggered — never let a retry
|
|
331
|
+
* depend on a future event." The spread check has no driver of its own: it runs
|
|
332
|
+
* post-fill and on the blockchain-fetch tick (240 min by default, and the
|
|
333
|
+
* open-orders loop that would shorten it is OFF by default). Every watchdog
|
|
334
|
+
* window (warn 10 min, geometry 10 min, fund 5 min, pivot-expiry 20 min) is
|
|
335
|
+
* therefore unreachable while the grid sits out of tolerance, which is how a
|
|
336
|
+
* live bot went 86 minutes without a single check. While out of tolerance the
|
|
337
|
+
* watchdog now schedules its own retry, so the clocks run at any base cadence.
|
|
338
|
+
*
|
|
339
|
+
* Deduped and shutdown-safe: one timer at a time, never while a recheck is in
|
|
340
|
+
* flight, never while shutting down. `performPeriodicGridChecks` keeps its own
|
|
341
|
+
* pipeline/broadcast deferrals, so this cannot interleave with a fill cycle.
|
|
342
|
+
*
|
|
343
|
+
* @param {Object} bot - DEXBot instance
|
|
344
|
+
* @param {number} [delayOverrideMs] - Injectable delay (tests); production reads TIMING
|
|
345
|
+
* @returns {boolean} True when a recheck was armed (or already pending)
|
|
346
|
+
*/
|
|
347
|
+
export declare function scheduleSpreadRecheck(bot: BotLike, delayOverrideMs?: number): boolean;
|
|
327
348
|
/**
|
|
328
349
|
* Out-of-spread persistence watchdog: the never-run-stale backstop.
|
|
329
350
|
*
|
|
@@ -342,9 +363,12 @@ declare function executeMaintenanceLogic(bot: BotLike, context: string): Promise
|
|
|
342
363
|
* @param {number} ordersPlaced - Correction orders placed this tick
|
|
343
364
|
* @param {boolean} [transient=false] - True when the correction was refused for
|
|
344
365
|
* a retryable reason (stale plan / duplicate CREATE / unclassified non-execution)
|
|
366
|
+
* @param {number} [excessAfter] - Out-of-spread violation (grid steps) MEASURED
|
|
367
|
+
* after the placement, or undefined when the correction placed nothing /
|
|
368
|
+
* could not be measured
|
|
345
369
|
* @returns {{staleMs: number, escalated: boolean}}
|
|
346
370
|
*/
|
|
347
|
-
export declare function trackOutOfSpreadStaleness(bot: BotLike, spreadChecked: boolean, ordersPlaced: number, transient?: boolean): {
|
|
371
|
+
export declare function trackOutOfSpreadStaleness(bot: BotLike, spreadChecked: boolean, ordersPlaced: number, transient?: boolean, excessAfter?: number): {
|
|
348
372
|
staleMs: number;
|
|
349
373
|
escalated: boolean;
|
|
350
374
|
};
|
|
@@ -461,6 +485,11 @@ declare function getMetrics(bot: BotLike): {
|
|
|
461
485
|
spreadCorrectionRefusedCount: number;
|
|
462
486
|
spreadCorrectionRefusalCauses: Record<string, number>;
|
|
463
487
|
spreadCorrectionRetryCount: number;
|
|
488
|
+
stalePlanDroppedCount: number;
|
|
489
|
+
stalePlanProceedCount: number;
|
|
490
|
+
clampedNoopUpdateCount: number;
|
|
491
|
+
chunkAbortedOpCount: number;
|
|
492
|
+
reloadReplacedRailCount: number;
|
|
464
493
|
};
|
|
465
494
|
/**
|
|
466
495
|
* Read open orders from chain, sync with local state, and process any fills found.
|
|
@@ -492,6 +521,7 @@ declare const _default: {
|
|
|
492
521
|
handlePendingTriggerReset: typeof handlePendingTriggerReset;
|
|
493
522
|
setupTriggerFileDetection: typeof setupTriggerFileDetection;
|
|
494
523
|
performPeriodicGridChecks: typeof performPeriodicGridChecks;
|
|
524
|
+
scheduleSpreadRecheck: typeof scheduleSpreadRecheck;
|
|
495
525
|
isOpenOrdersSyncLoopEnabled: typeof isOpenOrdersSyncLoopEnabled;
|
|
496
526
|
startOpenOrdersSyncLoop: typeof startOpenOrdersSyncLoop;
|
|
497
527
|
stopOpenOrdersSyncLoop: typeof stopOpenOrdersSyncLoop;
|