dexbot 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/credential-daemon.d.ts.map +1 -1
- package/dist/credential-daemon.js +53 -13
- package/dist/credential-daemon.js.map +1 -1
- package/dist/modules/bitshares-native/serial/types.js +1 -1
- package/dist/modules/bitshares-native/serial/types.js.map +1 -1
- package/dist/modules/chain_keys.js +1 -1
- package/dist/modules/chain_keys.js.map +1 -1
- package/dist/modules/constants.d.ts +2 -0
- package/dist/modules/constants.d.ts.map +1 -1
- package/dist/modules/constants.js +10 -0
- package/dist/modules/constants.js.map +1 -1
- package/dist/modules/credential_policy.js +1 -1
- package/dist/modules/credential_policy.js.map +1 -1
- package/dist/modules/dexbot_class.d.ts +45 -175
- package/dist/modules/dexbot_class.d.ts.map +1 -1
- package/dist/modules/dexbot_class.js +84 -3496
- package/dist/modules/dexbot_class.js.map +1 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +319 -0
- package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -0
- package/dist/modules/dexbot_cow_runtime.js +1764 -0
- package/dist/modules/dexbot_cow_runtime.js.map +1 -0
- package/dist/modules/dexbot_fill_runtime.d.ts +47 -0
- package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_fill_runtime.js +607 -3
- package/dist/modules/dexbot_fill_runtime.js.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts +69 -0
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +244 -15
- package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
- package/dist/modules/dexbot_startup_runtime.d.ts +30 -0
- package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -0
- package/dist/modules/dexbot_startup_runtime.js +539 -0
- package/dist/modules/dexbot_startup_runtime.js.map +1 -0
- package/dist/modules/dexbot_state_recovery.d.ts +128 -0
- package/dist/modules/dexbot_state_recovery.d.ts.map +1 -0
- package/dist/modules/dexbot_state_recovery.js +397 -0
- package/dist/modules/dexbot_state_recovery.js.map +1 -0
- package/dist/modules/fund_registry.d.ts.map +1 -1
- package/dist/modules/fund_registry.js +0 -1
- package/dist/modules/fund_registry.js.map +1 -1
- package/dist/modules/general_settings.d.ts.map +1 -1
- package/dist/modules/general_settings.js +0 -1
- package/dist/modules/general_settings.js.map +1 -1
- package/dist/modules/key_store.d.ts.map +1 -1
- package/dist/modules/key_store.js +1 -2
- package/dist/modules/key_store.js.map +1 -1
- package/dist/modules/launcher/market_adapter_runtime.d.ts.map +1 -1
- package/dist/modules/launcher/market_adapter_runtime.js +0 -1
- package/dist/modules/launcher/market_adapter_runtime.js.map +1 -1
- package/dist/modules/launcher/market_adapter_watchdog.d.ts.map +1 -1
- package/dist/modules/launcher/market_adapter_watchdog.js +1 -2
- package/dist/modules/launcher/market_adapter_watchdog.js.map +1 -1
- package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile.js +0 -1
- package/dist/modules/order/grid_reconcile.js.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.js +2 -2
- package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
- package/dist/modules/order/utils/math.js +1 -1
- package/dist/modules/order/utils/math.js.map +1 -1
- package/dist/modules/order/utils/system.js +1 -1
- package/dist/modules/order/utils/system.js.map +1 -1
- package/dist/modules/order/utils/validate.d.ts.map +1 -1
- package/dist/modules/order/utils/validate.js +0 -1
- package/dist/modules/order/utils/validate.js.map +1 -1
- package/dist/modules/paths.d.ts +2 -0
- package/dist/modules/paths.d.ts.map +1 -1
- package/dist/modules/paths.js +13 -2
- package/dist/modules/paths.js.map +1 -1
- package/dist/scripts/verify-browser-bundle.js +25 -1
- package/dist/scripts/verify-browser-bundle.js.map +1 -1
- package/package.json +7 -2
|
@@ -53,33 +53,26 @@
|
|
|
53
53
|
*
|
|
54
54
|
* ===============================================================================
|
|
55
55
|
*/
|
|
56
|
-
const {
|
|
57
|
-
const { BitShares, waitForConnected, onReconnect: registerReconnectHook } = require('./bitshares_client');
|
|
58
|
-
const { getStorage } = require('./storage');
|
|
59
|
-
const storage = getStorage();
|
|
56
|
+
const { BitShares } = require('./bitshares_client');
|
|
60
57
|
const chainKeys = require('./chain_keys');
|
|
61
|
-
const { getKeyStore } = require('./key_store');
|
|
62
58
|
const chainOrders = require('./chain_orders');
|
|
63
59
|
const fundRegistry = require('./fund_registry');
|
|
64
|
-
const { BroadcastUncertainError } = require('./dexbot_credential_client');
|
|
65
|
-
const { OrderManager, grid: Grid } = require('./order');
|
|
66
|
-
const { retryPersistenceIfNeeded, initializeFeeCache, } = require('./order/utils/system');
|
|
67
|
-
const { hasExecutableActions, validateCreateTargetSlots } = require('./order/utils/validate');
|
|
68
|
-
const { buildCreateOrderArgs, buildCreateOpFingerprint, virtualizeOrder, correctAllPriceMismatches, convertToSpreadPlaceholder, buildOutsideInPairGroups, extractBatchOperationResults, buildFillKey, formatUnmatchedChainOrder, parseChainOrder } = require('./order/utils/order');
|
|
69
|
-
const { validateOrderSize } = require('./order/utils/math');
|
|
70
60
|
const { ProcessedFillStore, PROCESSED_FILL_PERSISTENCE_MODES } = require('./order/processed_fill_store');
|
|
71
61
|
const DexbotFillRuntime = require('./dexbot_fill_runtime');
|
|
72
62
|
const DexbotMaintenanceRuntime = require('./dexbot_maintenance_runtime');
|
|
63
|
+
const DexbotStateRecovery = require('./dexbot_state_recovery');
|
|
64
|
+
const DexbotStartupRuntime = require('./dexbot_startup_runtime');
|
|
73
65
|
const CreditRuntime = require('./credit_runtime');
|
|
74
|
-
const {
|
|
75
|
-
const { PATHS
|
|
76
|
-
const { attemptResumePersistedGridByPriceMatch, decideStartupGridAction, reconcileGridOrders } = require('./order/grid_reconcile');
|
|
77
|
-
const { AccountOrders } = require('./account_orders');
|
|
78
|
-
const { parseJsonWithComments } = require('./order/utils/system');
|
|
79
|
-
const { cloneWeightDistribution } = require('./order/utils/math');
|
|
80
|
-
const { normalizeBotEntry } = require('./bot_settings');
|
|
66
|
+
const { TIMING, FILL_PROCESSING, DAEMON_CODES, } = require('./constants');
|
|
67
|
+
const { PATHS } = require('./paths');
|
|
81
68
|
const Format = require('./order/format');
|
|
82
|
-
const
|
|
69
|
+
const cowRuntime = require('./dexbot_cow_runtime');
|
|
70
|
+
function waitForConnected(...args) { return require('./bitshares_client').waitForConnected(...args); }
|
|
71
|
+
function getKeyStore(...args) { return require('./key_store').getKeyStore(...args); }
|
|
72
|
+
function hasExecutableActions(...args) { return require('./order/utils/validate').hasExecutableActions(...args); }
|
|
73
|
+
function getRecalculateTriggerFile(...args) { return require('./paths').getRecalculateTriggerFile(...args); }
|
|
74
|
+
function cloneWeightDistribution(...args) { return require('./order/utils/math').cloneWeightDistribution(...args); }
|
|
75
|
+
function resolveBotRuntimeSettings(...args) { return require('./runtime_settings').resolveBotRuntimeSettings(...args); }
|
|
83
76
|
const PROFILES_BOTS_FILE = PATHS.PROFILES.BOTS_JSON;
|
|
84
77
|
const PROFILES_DIR = PATHS.PROFILES_DIR;
|
|
85
78
|
class DEXBot {
|
|
@@ -324,21 +317,7 @@ class DEXBot {
|
|
|
324
317
|
* @private
|
|
325
318
|
*/
|
|
326
319
|
async _persistAndRecoverIfNeeded() {
|
|
327
|
-
|
|
328
|
-
const validation = await this.manager.persistGrid();
|
|
329
|
-
if (!validation.isValid) {
|
|
330
|
-
this._warn(`Startup validation failed: ${validation.reason}. Triggering immediate recovery...`);
|
|
331
|
-
// Trigger centralized recovery (Hard Reset)
|
|
332
|
-
const recoveryValidation = await this.manager.accountant._performStateRecovery(this.manager);
|
|
333
|
-
if (recoveryValidation.isValid) {
|
|
334
|
-
this._log(`✓ Startup recovery successful. Persistent state restored.`);
|
|
335
|
-
this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
|
|
336
|
-
await this.manager.persistGrid();
|
|
337
|
-
}
|
|
338
|
-
else {
|
|
339
|
-
this._warn(`Startup recovery failed: ${recoveryValidation.reason}. Bot proceeding with caution.`);
|
|
340
|
-
}
|
|
341
|
-
}
|
|
320
|
+
return DexbotStateRecovery.persistAndRecoverIfNeeded(this);
|
|
342
321
|
}
|
|
343
322
|
/**
|
|
344
323
|
* Snapshot the recently queued fill keys as a plain object for crash-durable persistence.
|
|
@@ -348,42 +327,14 @@ class DEXBot {
|
|
|
348
327
|
* @returns {Record<string, number>}
|
|
349
328
|
*/
|
|
350
329
|
_getRecentFillKeysSnapshot() {
|
|
351
|
-
|
|
352
|
-
const now = Date.now();
|
|
353
|
-
// 1. Collect live keys from the in-memory map
|
|
354
|
-
for (const [key, timestamp] of this._recentlyQueuedFills) {
|
|
355
|
-
if (now - Number(timestamp) < this._fillDedupeWindowMs) {
|
|
356
|
-
snapshot[key] = Number(timestamp);
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
// 2. Merge with the previous snapshot — carry forward any keys that
|
|
360
|
-
// were present in a prior persist cycle, are still within the
|
|
361
|
-
// dedup window, but have been evicted from _recentlyQueuedFills
|
|
362
|
-
// (e.g. by TTL-based pruning between two persistGrid calls).
|
|
363
|
-
// This prevents a crash + restart from losing keys that were
|
|
364
|
-
// processed but not yet persisted.
|
|
365
|
-
if (this.manager?._recentFillKeysSnapshot) {
|
|
366
|
-
for (const [key, timestamp] of Object.entries(this.manager._recentFillKeysSnapshot)) {
|
|
367
|
-
if (!(key in snapshot) && now - Number(timestamp) < this._fillDedupeWindowMs) {
|
|
368
|
-
snapshot[key] = Number(timestamp);
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
return snapshot;
|
|
330
|
+
return DexbotStateRecovery.getRecentFillKeysSnapshot(this);
|
|
373
331
|
}
|
|
374
332
|
/**
|
|
375
333
|
* Get current pipeline signal state for congestion checks.
|
|
376
334
|
* @returns {{incomingFillQueueLength: number, shadowLocks: number, batchInFlight: boolean, recoveryInFlight: boolean, broadcasting: boolean}}
|
|
377
335
|
*/
|
|
378
336
|
_getPipelineSignals() {
|
|
379
|
-
|
|
380
|
-
return {
|
|
381
|
-
incomingFillQueueLength: this._incomingFillQueue.length,
|
|
382
|
-
shadowLocks: this.manager?.shadowOrderIds?.size || 0,
|
|
383
|
-
batchInFlight: this._batchInFlight,
|
|
384
|
-
recoveryInFlight: this._recoverySyncInFlight,
|
|
385
|
-
broadcasting: this.manager?.isBroadcastingActive?.() || false
|
|
386
|
-
};
|
|
337
|
+
return DexbotMaintenanceRuntime.getPipelineSignals(this);
|
|
387
338
|
}
|
|
388
339
|
/**
|
|
389
340
|
* Mark that grid activity occurred (updates idle timer).
|
|
@@ -391,8 +342,7 @@ class DEXBot {
|
|
|
391
342
|
* @returns {void}
|
|
392
343
|
*/
|
|
393
344
|
_markGridActivity(reason = 'activity') {
|
|
394
|
-
|
|
395
|
-
this.manager?.logger?.log?.(`[MAINT-IDLE] Activity observed: ${reason}`, 'debug');
|
|
345
|
+
return DexbotMaintenanceRuntime.markGridActivity(this, reason);
|
|
396
346
|
}
|
|
397
347
|
/**
|
|
398
348
|
* Trigger a full state recovery sync (fetch chain + sync from open orders + persist).
|
|
@@ -400,25 +350,7 @@ class DEXBot {
|
|
|
400
350
|
* @returns {Promise<void>}
|
|
401
351
|
*/
|
|
402
352
|
async _triggerStateRecoverySync(reason = 'state recovery sync') {
|
|
403
|
-
|
|
404
|
-
return;
|
|
405
|
-
if (this._recoverySyncInFlight) {
|
|
406
|
-
this.manager.logger.log(`[RECOVERY] Skipping duplicate recovery request: ${reason}`, 'warn');
|
|
407
|
-
return;
|
|
408
|
-
}
|
|
409
|
-
this._recoverySyncInFlight = true;
|
|
410
|
-
try {
|
|
411
|
-
this.manager.logger.log(`Triggering state recovery sync (${reason})...`, 'info');
|
|
412
|
-
await this.manager.fetchAccountTotals(this.accountId);
|
|
413
|
-
const openOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
414
|
-
await this.manager.syncFromOpenOrders(openOrders, { skipAccounting: true });
|
|
415
|
-
if (typeof this.manager.persistGrid === 'function') {
|
|
416
|
-
await this.manager.persistGrid();
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
finally {
|
|
420
|
-
this._recoverySyncInFlight = false;
|
|
421
|
-
}
|
|
353
|
+
return DexbotStateRecovery.triggerStateRecoverySync(this, reason);
|
|
422
354
|
}
|
|
423
355
|
/**
|
|
424
356
|
* Abort the current flow if an illegal state signal was raised.
|
|
@@ -426,14 +358,7 @@ class DEXBot {
|
|
|
426
358
|
* @returns {Promise<boolean>} True if flow was aborted
|
|
427
359
|
*/
|
|
428
360
|
async _abortFlowIfIllegalState(flowContext) {
|
|
429
|
-
|
|
430
|
-
if (!illegalSignal) {
|
|
431
|
-
return false;
|
|
432
|
-
}
|
|
433
|
-
this.manager.logger.log(`[HARD-ABORT] ${flowContext} aborted due to illegal state (${illegalSignal.context}): ${illegalSignal.message}`, 'error');
|
|
434
|
-
await this._triggerStateRecoverySync(`hard-abort ${flowContext}`);
|
|
435
|
-
this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
|
|
436
|
-
return true;
|
|
361
|
+
return DexbotStateRecovery.abortFlowIfIllegalState(this, flowContext);
|
|
437
362
|
}
|
|
438
363
|
/**
|
|
439
364
|
* Handle a hard abort from batch processing due to illegal state or accounting failure.
|
|
@@ -443,24 +368,7 @@ class DEXBot {
|
|
|
443
368
|
* @returns {Promise<Object>} Abort result object
|
|
444
369
|
*/
|
|
445
370
|
async _handleBatchHardAbort(err, phase = 'batch processing', opsCount = 0) {
|
|
446
|
-
|
|
447
|
-
const opsInfo = opsCount > 0 ? ` with ${opsCount} ops` : '';
|
|
448
|
-
if (err?.code === 'ILLEGAL_ORDER_STATE') {
|
|
449
|
-
const illegalSignal = this.manager.consumeIllegalStateSignal?.();
|
|
450
|
-
await this._triggerStateRecoverySync(illegalSignal?.message || `illegal order state during ${phase}${opsInfo}`);
|
|
451
|
-
this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
|
|
452
|
-
return { ...baseResult, abortedForIllegalState: true };
|
|
453
|
-
}
|
|
454
|
-
if (err?.code === 'ACCOUNTING_COMMITMENT_FAILED') {
|
|
455
|
-
const accountingSignal = this.manager.consumeAccountingFailureSignal?.();
|
|
456
|
-
const reason = accountingSignal
|
|
457
|
-
? `accounting lock failure (${accountingSignal.side} ${Format.formatAmount8(accountingSignal.amount)}) during ${accountingSignal.context}`
|
|
458
|
-
: `accounting commitment lock failure during ${phase}${opsInfo}`;
|
|
459
|
-
await this._triggerStateRecoverySync(reason);
|
|
460
|
-
this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
|
|
461
|
-
return { ...baseResult, abortedForAccountingFailure: true };
|
|
462
|
-
}
|
|
463
|
-
return null;
|
|
371
|
+
return DexbotStateRecovery.handleBatchHardAbort(this, err, phase, opsCount);
|
|
464
372
|
}
|
|
465
373
|
/**
|
|
466
374
|
* Apply recoverable grid updates (order virtualisation) after a batch failure.
|
|
@@ -469,30 +377,7 @@ class DEXBot {
|
|
|
469
377
|
* @returns {Promise<number>} Number of updates applied
|
|
470
378
|
*/
|
|
471
379
|
async _applyRecoverableGridUpdates(updates, context = 'recoverable-grid-update') {
|
|
472
|
-
|
|
473
|
-
return 0;
|
|
474
|
-
}
|
|
475
|
-
let applied;
|
|
476
|
-
if (typeof this.manager.applyGridUpdateBatch === 'function') {
|
|
477
|
-
await this.manager.applyGridUpdateBatch(updates, context);
|
|
478
|
-
applied = updates.length;
|
|
479
|
-
}
|
|
480
|
-
else {
|
|
481
|
-
applied = 0;
|
|
482
|
-
for (const update of updates) {
|
|
483
|
-
if (typeof this.manager._updateOrder !== 'function')
|
|
484
|
-
break;
|
|
485
|
-
await this.manager._updateOrder(update, context);
|
|
486
|
-
applied++;
|
|
487
|
-
}
|
|
488
|
-
}
|
|
489
|
-
// Persist master grid mutations applied outside COW (stale-order
|
|
490
|
-
// virtualization, size-drift corrections). These run in the COW catch
|
|
491
|
-
// handler where the success-path persistGrid is never reached.
|
|
492
|
-
if (applied > 0 && typeof this.manager.persistGrid === 'function') {
|
|
493
|
-
await this.manager.persistGrid();
|
|
494
|
-
}
|
|
495
|
-
return applied;
|
|
380
|
+
return DexbotStateRecovery.applyRecoverableGridUpdates(this, updates, context);
|
|
496
381
|
}
|
|
497
382
|
/**
|
|
498
383
|
* Recover from explicit stale order errors by virtualizing affected grid slots.
|
|
@@ -501,36 +386,7 @@ class DEXBot {
|
|
|
501
386
|
* @returns {Promise<{executed: boolean, hadRotation: boolean, stale: boolean, recoveredByVirtualization?: boolean}>}
|
|
502
387
|
*/
|
|
503
388
|
async _recoverExplicitStaleOrders(staleOrderIds, reason = 'stale order cleanup') {
|
|
504
|
-
|
|
505
|
-
if (staleIds.length === 0) {
|
|
506
|
-
return { executed: false, hadRotation: false, stale: false };
|
|
507
|
-
}
|
|
508
|
-
this.manager.logger.log(`[COW] Stale order(s) detected: ${staleIds.join(', ')}. Applying targeted cleanup.`, 'warn');
|
|
509
|
-
const updates = [];
|
|
510
|
-
for (const [, gridOrder] of this.manager.orders.entries()) {
|
|
511
|
-
if (!gridOrder?.orderId || !staleOrderIds.has(gridOrder.orderId))
|
|
512
|
-
continue;
|
|
513
|
-
this._staleCleanedOrderIds.set(gridOrder.orderId, Date.now());
|
|
514
|
-
updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
|
|
515
|
-
}
|
|
516
|
-
// Register any stale IDs that had no matching grid slot
|
|
517
|
-
for (const orderId of staleIds) {
|
|
518
|
-
if (!this._staleCleanedOrderIds.has(orderId)) {
|
|
519
|
-
this._staleCleanedOrderIds.set(orderId, Date.now());
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
if (updates.length > 0) {
|
|
523
|
-
await this._applyRecoverableGridUpdates(updates, reason);
|
|
524
|
-
}
|
|
525
|
-
else {
|
|
526
|
-
this.manager.logger.log(`[COW] No local grid slot matched stale order cleanup request (${staleIds.join(', ')}).`, 'debug');
|
|
527
|
-
}
|
|
528
|
-
return {
|
|
529
|
-
executed: false,
|
|
530
|
-
hadRotation: false,
|
|
531
|
-
stale: true,
|
|
532
|
-
recoveredByVirtualization: updates.length > 0
|
|
533
|
-
};
|
|
389
|
+
return DexbotStateRecovery.recoverExplicitStaleOrders(this, staleOrderIds, reason);
|
|
534
390
|
}
|
|
535
391
|
/**
|
|
536
392
|
* Recover from on-chain size drift detected during batch broadcast.
|
|
@@ -538,32 +394,7 @@ class DEXBot {
|
|
|
538
394
|
* @returns {Promise<{executed: boolean, hadRotation: boolean, recoveredBySync: boolean, reason: string}>}
|
|
539
395
|
*/
|
|
540
396
|
async _recoverBatchSizeDrift(err, opContexts = []) {
|
|
541
|
-
|
|
542
|
-
// operation contexts and correct them directly from chain. This
|
|
543
|
-
// avoids a full state recovery sync in the common single-order case.
|
|
544
|
-
const affectedOrderIds = this._extractSizeDriftOrderIds(opContexts);
|
|
545
|
-
if (affectedOrderIds.length > 0) {
|
|
546
|
-
this.manager.logger.log(`[COW] Targeted size-drift repair for ${affectedOrderIds.length} order(s): ${affectedOrderIds.join(', ')}`, 'debug');
|
|
547
|
-
const repaired = await this._targetedOrderRepair(affectedOrderIds);
|
|
548
|
-
if (repaired) {
|
|
549
|
-
return {
|
|
550
|
-
executed: false,
|
|
551
|
-
hadRotation: false,
|
|
552
|
-
recoveredBySync: true,
|
|
553
|
-
reason: 'ORDER_SIZE_DRIFT_TARGETED'
|
|
554
|
-
};
|
|
555
|
-
}
|
|
556
|
-
this.manager.logger.log('[COW] Targeted repair failed, falling back to full state recovery sync.', 'warn');
|
|
557
|
-
}
|
|
558
|
-
const reason = `recoverable size drift during COW batch: ${err.message}`;
|
|
559
|
-
this.manager.logger.log(`[COW] Recovering from on-chain size drift via recovery sync: ${err.message}`, 'warn');
|
|
560
|
-
await this._triggerStateRecoverySync(reason);
|
|
561
|
-
return {
|
|
562
|
-
executed: false,
|
|
563
|
-
hadRotation: false,
|
|
564
|
-
recoveredBySync: true,
|
|
565
|
-
reason: 'ORDER_SIZE_DRIFT'
|
|
566
|
-
};
|
|
397
|
+
return DexbotStateRecovery.recoverBatchSizeDrift(this, err, opContexts);
|
|
567
398
|
}
|
|
568
399
|
/**
|
|
569
400
|
* Extract chain order IDs from opContexts for operations that could
|
|
@@ -572,18 +403,7 @@ class DEXBot {
|
|
|
572
403
|
* @returns {string[]} Unique chain order IDs
|
|
573
404
|
*/
|
|
574
405
|
_extractSizeDriftOrderIds(opContexts) {
|
|
575
|
-
|
|
576
|
-
return [];
|
|
577
|
-
const ids = new Set();
|
|
578
|
-
for (const ctx of opContexts) {
|
|
579
|
-
if (ctx?.kind === 'size-update' && ctx?.updateInfo?.partialOrder?.orderId) {
|
|
580
|
-
ids.add(ctx.updateInfo.partialOrder.orderId);
|
|
581
|
-
}
|
|
582
|
-
else if (ctx?.kind === 'rotation' && ctx?.rotation?.oldOrder?.orderId) {
|
|
583
|
-
ids.add(ctx.rotation.oldOrder.orderId);
|
|
584
|
-
}
|
|
585
|
-
}
|
|
586
|
-
return Array.from(ids);
|
|
406
|
+
return DexbotStateRecovery.extractSizeDriftOrderIds(opContexts);
|
|
587
407
|
}
|
|
588
408
|
/**
|
|
589
409
|
* Reload the entire grid from the persisted on-disk snapshot and reconcile
|
|
@@ -596,87 +416,7 @@ class DEXBot {
|
|
|
596
416
|
* @returns {Promise<{success: boolean, reason?: string}>}
|
|
597
417
|
*/
|
|
598
418
|
async _recoverFromPersistedGrid() {
|
|
599
|
-
|
|
600
|
-
return { success: false, reason: 'accountOrders or manager unavailable' };
|
|
601
|
-
}
|
|
602
|
-
const accountRef = this.accountId || this.account?.id || this.account;
|
|
603
|
-
if (!accountRef) {
|
|
604
|
-
return { success: false, reason: 'no account reference' };
|
|
605
|
-
}
|
|
606
|
-
this.manager.logger.log('[RECOVERY] Attempting full grid reload from persisted snapshot...', 'warn');
|
|
607
|
-
try {
|
|
608
|
-
// 1. Force reload from disk
|
|
609
|
-
const persistedGrid = this.accountOrders.loadGrid(true);
|
|
610
|
-
if (!persistedGrid || persistedGrid.length === 0) {
|
|
611
|
-
return { success: false, reason: 'no persisted grid on disk' };
|
|
612
|
-
}
|
|
613
|
-
const boundaryIdx = this.accountOrders.loadBoundaryIdx(true);
|
|
614
|
-
// 2. Load into manager (same path as startup at line 1340)
|
|
615
|
-
await Grid.loadGrid(this.manager, persistedGrid, boundaryIdx);
|
|
616
|
-
// Gap 1: Grid snapshot sanity check — shared logic with startup path.
|
|
617
|
-
if (await this._rejectCorruptedGridSnapshot('recovery')) {
|
|
618
|
-
return { success: false, reason: 'corrupted grid snapshot rejected (fund drift)' };
|
|
619
|
-
}
|
|
620
|
-
// 3. Read current chain state
|
|
621
|
-
const chainOpenOrders = await chainOrders.readOpenOrders(accountRef);
|
|
622
|
-
// 4. Reconcile
|
|
623
|
-
if (chainOpenOrders.length > 0 && this.manager?.syncFromOpenOrders) {
|
|
624
|
-
await this.manager.syncFromOpenOrders(chainOpenOrders, {
|
|
625
|
-
skipAccounting: true,
|
|
626
|
-
});
|
|
627
|
-
}
|
|
628
|
-
// 5. Persist the reconciled state
|
|
629
|
-
if (typeof this.manager.persistGrid === 'function') {
|
|
630
|
-
await this.manager.persistGrid();
|
|
631
|
-
}
|
|
632
|
-
const assets = this.manager?.assets;
|
|
633
|
-
const matchedCount = assets
|
|
634
|
-
? chainOpenOrders.filter(o => parseChainOrder(o, assets) !== null).length
|
|
635
|
-
: chainOpenOrders.length;
|
|
636
|
-
this.manager.logger.log(`[RECOVERY] Grid reloaded from persisted snapshot: ${this.manager.orders.size} orders, ` +
|
|
637
|
-
`${matchedCount} on-chain orders synced`, 'info');
|
|
638
|
-
// Gap 2: Check for unmatched chain orders after reload + sync.
|
|
639
|
-
// If the sync resolved all unmatched entries, _lastUnmatchedChainOrders
|
|
640
|
-
// was cleared by the sync engine. If any remain, the persisted snapshot
|
|
641
|
-
// produced an inconsistent grid — reject so the structural resync
|
|
642
|
-
// falls through to requestGridReset (full rebuild from chain).
|
|
643
|
-
const remainingUnmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
|
|
644
|
-
? this.manager._lastUnmatchedChainOrders
|
|
645
|
-
: [];
|
|
646
|
-
if (remainingUnmatched.length > 0) {
|
|
647
|
-
const sample = remainingUnmatched.slice(0, 3)
|
|
648
|
-
.map(o => this._formatUnmatchedChainOrderForLog(o))
|
|
649
|
-
.join(' | ');
|
|
650
|
-
this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but ${remainingUnmatched.length} unmatched chain order(s) ` +
|
|
651
|
-
`remain${sample ? ` (${sample})` : ''}. Rejecting — full grid reset required.`, 'warn');
|
|
652
|
-
return { success: false, reason: `grid inconsistent after reload: ${remainingUnmatched.length} unmatched remain` };
|
|
653
|
-
}
|
|
654
|
-
// If the reloaded grid is still bloated, reject recovery so the
|
|
655
|
-
// caller falls through to a full grid reset (requestGridReset).
|
|
656
|
-
// Without this check, a bloated snapshot gets accepted as "success"
|
|
657
|
-
// and the structural-resync loop loads the same broken state forever.
|
|
658
|
-
//
|
|
659
|
-
// NOTE: loadGrid() already fires requestStructuralGridResync when it
|
|
660
|
-
// detects bloat internally, so the inner async resync may be in flight
|
|
661
|
-
// by the time this outer check runs. That's fine — the structural-resync
|
|
662
|
-
// gate (_structuralGridResyncRunning / _structuralGridResyncTimer) dedup's
|
|
663
|
-
// concurrent requests. This outer check exists so the synchronous return
|
|
664
|
-
// value is honest about the state; the inner resync is a safety net.
|
|
665
|
-
const { isGridBloated } = require('./order/grid');
|
|
666
|
-
const ordersArr = Array.from(this.manager.orders.values());
|
|
667
|
-
const bloatPostRecovery = isGridBloated(this.manager, ordersArr);
|
|
668
|
-
if (bloatPostRecovery.bloated) {
|
|
669
|
-
const d = bloatPostRecovery.details;
|
|
670
|
-
this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but still bloated ` +
|
|
671
|
-
`(${d.gridSize} slots, max ${d.maxAllowed}). Rejecting — full grid reset required.`, 'warn');
|
|
672
|
-
return { success: false, reason: 'grid still bloated after reload' };
|
|
673
|
-
}
|
|
674
|
-
return { success: true };
|
|
675
|
-
}
|
|
676
|
-
catch (err) {
|
|
677
|
-
this.manager.logger.log(`[RECOVERY] Full grid reload from persisted snapshot failed: ${err.message}`, 'error');
|
|
678
|
-
return { success: false, reason: err.message };
|
|
679
|
-
}
|
|
419
|
+
return DexbotStateRecovery.recoverFromPersistedGrid(this);
|
|
680
420
|
}
|
|
681
421
|
/**
|
|
682
422
|
* Reject a corrupted grid snapshot when catastrophic fund drift is detected.
|
|
@@ -687,25 +427,7 @@ class DEXBot {
|
|
|
687
427
|
* @returns {Promise<boolean>} True if the snapshot was rejected (cleared).
|
|
688
428
|
*/
|
|
689
429
|
async _rejectCorruptedGridSnapshot(context) {
|
|
690
|
-
|
|
691
|
-
return false;
|
|
692
|
-
const driftCheck = this.manager.checkFundDriftAfterFills();
|
|
693
|
-
if (driftCheck.isValid)
|
|
694
|
-
return false;
|
|
695
|
-
const tag = context === 'recovery' ? '[RECOVERY][SNAPSHOT-REJECT]' : '[SNAPSHOT-REJECT]';
|
|
696
|
-
this._warn(`${tag} Corrupted grid snapshot detected: ` +
|
|
697
|
-
`drift sell=${driftCheck.driftSell.toFixed(2)} buy=${driftCheck.driftBuy.toFixed(2)}. ` +
|
|
698
|
-
`Deleting corrupted snapshot.`);
|
|
699
|
-
if (this.accountOrders && typeof this.accountOrders.clearGrid === 'function') {
|
|
700
|
-
try {
|
|
701
|
-
await this.accountOrders.clearGrid();
|
|
702
|
-
this._warn(`${tag} Corrupted grid snapshot deleted.`);
|
|
703
|
-
}
|
|
704
|
-
catch (clearErr) {
|
|
705
|
-
this._warn(`${tag} Failed to delete corrupted snapshot: ${clearErr.message}`);
|
|
706
|
-
}
|
|
707
|
-
}
|
|
708
|
-
return true;
|
|
430
|
+
return DexbotStateRecovery.rejectCorruptedGridSnapshot(this, context);
|
|
709
431
|
}
|
|
710
432
|
/**
|
|
711
433
|
* Attempt to repair size-drift for specific order IDs by reading their
|
|
@@ -715,141 +437,16 @@ class DEXBot {
|
|
|
715
437
|
* @returns {Promise<boolean>} True if all affected orders were repaired
|
|
716
438
|
*/
|
|
717
439
|
async _targetedOrderRepair(orderIds) {
|
|
718
|
-
|
|
719
|
-
const objects = await BitShares.db.get_objects(orderIds);
|
|
720
|
-
if (!Array.isArray(objects) || objects.length !== orderIds.length)
|
|
721
|
-
return false;
|
|
722
|
-
const updates = [];
|
|
723
|
-
for (let i = 0; i < orderIds.length; i++) {
|
|
724
|
-
const chainOrder = objects[i];
|
|
725
|
-
const gridOrder = Array.from(this.manager.orders.values())
|
|
726
|
-
.find((o) => o.orderId === orderIds[i]);
|
|
727
|
-
if (!gridOrder)
|
|
728
|
-
continue;
|
|
729
|
-
if (!chainOrder || typeof chainOrder.for_sale === 'undefined') {
|
|
730
|
-
// Order no longer exists on chain -> fully filled or cancelled.
|
|
731
|
-
updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
|
|
732
|
-
}
|
|
733
|
-
else {
|
|
734
|
-
const chainUnits = Number(chainOrder.for_sale);
|
|
735
|
-
if (Number.isFinite(chainUnits)) {
|
|
736
|
-
const { blockchainToFloat } = require('./order/utils/math');
|
|
737
|
-
const prec = gridOrder.type === ORDER_TYPES.SELL
|
|
738
|
-
? this.manager.assets.assetA.precision
|
|
739
|
-
: this.manager.assets.assetB.precision;
|
|
740
|
-
const floatSize = blockchainToFloat(chainUnits, prec);
|
|
741
|
-
if (floatSize !== gridOrder.size) {
|
|
742
|
-
updates.push({
|
|
743
|
-
id: gridOrder.id,
|
|
744
|
-
size: floatSize,
|
|
745
|
-
rawOnChain: chainOrder,
|
|
746
|
-
});
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
}
|
|
750
|
-
}
|
|
751
|
-
if (updates.length > 0) {
|
|
752
|
-
await this._applyRecoverableGridUpdates(updates, 'targeted-size-drift-repair');
|
|
753
|
-
}
|
|
754
|
-
return true;
|
|
755
|
-
}
|
|
756
|
-
catch (err) {
|
|
757
|
-
this.manager.logger.log(`[COW] Targeted order repair failed: ${err.message}`, 'debug');
|
|
758
|
-
return false;
|
|
759
|
-
}
|
|
440
|
+
return DexbotStateRecovery.targetedOrderRepair(this, orderIds);
|
|
760
441
|
}
|
|
761
442
|
/**
|
|
762
443
|
* Initialize bot state from storage and blockchain.
|
|
763
444
|
* Consolidates common initialization logic for start() and startWithPrivateKey().
|
|
764
|
-
* @returns {{persistedGrid: Object, persistedBtsFeesOwed: number, persistedBoundaryIdx: number, persistedBtsBalance: number}}
|
|
445
|
+
* @returns {{persistedGrid: Object, persistedBtsFeesOwed: number, persistedBoundaryIdx: number, persistedBtsBalance: number, persistedRecentFillKeys: Object}}
|
|
765
446
|
* @private
|
|
766
447
|
*/
|
|
767
448
|
async _initializeStartupState() {
|
|
768
|
-
|
|
769
|
-
this.accountOrders = new AccountOrders({ botKey: this.config.botKey });
|
|
770
|
-
this._processedFillStore.configure({
|
|
771
|
-
accountOrders: this.accountOrders
|
|
772
|
-
});
|
|
773
|
-
// Load persisted processed fills to prevent reprocessing after restart
|
|
774
|
-
const loadedPersistedFills = this._processedFillStore.loadPersisted({
|
|
775
|
-
minTimestamp: Date.now() - this._fillRecordRetentionMs
|
|
776
|
-
});
|
|
777
|
-
if (loadedPersistedFills > 0) {
|
|
778
|
-
this._log(`Loaded ${loadedPersistedFills} persisted fill records to prevent reprocessing`);
|
|
779
|
-
}
|
|
780
|
-
// Ensure bot metadata is properly initialized in storage BEFORE any Grid operations
|
|
781
|
-
const raw = storage.readFile(PROFILES_BOTS_FILE);
|
|
782
|
-
const allBotsConfig = parseJsonWithComments(raw).bots || [];
|
|
783
|
-
const myBotConfig = allBotsConfig
|
|
784
|
-
.map((b, originalIdx) => b.active !== false ? normalizeBotEntry(b, originalIdx) : null)
|
|
785
|
-
.find(b => b && b.botKey === this.config.botKey);
|
|
786
|
-
if (myBotConfig) {
|
|
787
|
-
await this.accountOrders.syncMeta(myBotConfig);
|
|
788
|
-
}
|
|
789
|
-
if (!this.manager) {
|
|
790
|
-
const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
|
|
791
|
-
this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
|
|
792
|
-
this.manager.account = this.account;
|
|
793
|
-
this.manager.accountId = this.accountId;
|
|
794
|
-
this.manager.accountOrders = this.accountOrders;
|
|
795
|
-
}
|
|
796
|
-
this._wireStructuralGridResyncRequest();
|
|
797
|
-
this._wireProcessedFillTracking();
|
|
798
|
-
this.manager.startBootstrap();
|
|
799
|
-
try {
|
|
800
|
-
// Fetch account totals from blockchain at startup to initialize funds
|
|
801
|
-
try {
|
|
802
|
-
if (this.accountId && this.config.assetA && this.config.assetB) {
|
|
803
|
-
await this.manager._initializeAssets();
|
|
804
|
-
await this.manager.fetchAccountTotals(this.accountId);
|
|
805
|
-
this._log('Fetched blockchain account balances at startup');
|
|
806
|
-
}
|
|
807
|
-
}
|
|
808
|
-
catch (err) {
|
|
809
|
-
this._log(`Startup balance fetch FAILED: ${err.message}. Order sizing may be incorrect until next successful sync.`, 'error');
|
|
810
|
-
}
|
|
811
|
-
// Ensure fee cache is initialized before any fill processing that calls getAssetFees().
|
|
812
|
-
try {
|
|
813
|
-
await initializeFeeCache([this.config || {}], BitShares);
|
|
814
|
-
}
|
|
815
|
-
catch (err) {
|
|
816
|
-
this._log(`Fee cache initialization FAILED: ${err.message}. Fee calculations will use defaults until cache is refreshed.`, 'error');
|
|
817
|
-
}
|
|
818
|
-
const persistedGrid = this.accountOrders.loadGrid();
|
|
819
|
-
// CRITICAL REPAIR: Strip fake orderIds where orderId === id (e.g. "slot-0")
|
|
820
|
-
let repairedGrid = persistedGrid;
|
|
821
|
-
if (persistedGrid && persistedGrid.length > 0) {
|
|
822
|
-
let repairCount = 0;
|
|
823
|
-
repairedGrid = persistedGrid.map(order => {
|
|
824
|
-
if (order && order.orderId && order.orderId === order.id) {
|
|
825
|
-
repairCount++;
|
|
826
|
-
const repairedOrder = { ...order, orderId: '' };
|
|
827
|
-
if (repairedOrder.state === ORDER_STATES.ACTIVE || repairedOrder.state === ORDER_STATES.PARTIAL) {
|
|
828
|
-
repairedOrder.state = ORDER_STATES.VIRTUAL;
|
|
829
|
-
}
|
|
830
|
-
return repairedOrder;
|
|
831
|
-
}
|
|
832
|
-
return order;
|
|
833
|
-
});
|
|
834
|
-
if (repairCount > 0) {
|
|
835
|
-
this._log(`[REPAIR] Stripped ${repairCount} fake orderId(s) from persisted grid to restore rebalancing logic.`);
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
const persistedBtsFeesOwed = this.accountOrders.loadBtsFeesOwed();
|
|
839
|
-
const persistedBoundaryIdx = this.accountOrders.loadBoundaryIdx();
|
|
840
|
-
const persistedBtsBalance = this.accountOrders.loadBtsBalance();
|
|
841
|
-
const persistedRecentFillKeys = this.accountOrders.loadRecentFillKeys();
|
|
842
|
-
return {
|
|
843
|
-
persistedGrid: repairedGrid,
|
|
844
|
-
persistedBtsFeesOwed,
|
|
845
|
-
persistedBoundaryIdx,
|
|
846
|
-
persistedBtsBalance,
|
|
847
|
-
persistedRecentFillKeys,
|
|
848
|
-
};
|
|
849
|
-
}
|
|
850
|
-
finally {
|
|
851
|
-
this.manager.finishBootstrap();
|
|
852
|
-
}
|
|
449
|
+
return DexbotStartupRuntime.initializeStartupState(this);
|
|
853
450
|
}
|
|
854
451
|
/**
|
|
855
452
|
* Wire processed fill tracking into the manager.
|
|
@@ -995,458 +592,7 @@ class DEXBot {
|
|
|
995
592
|
* @private
|
|
996
593
|
*/
|
|
997
594
|
async _finishStartupSequence(startupState) {
|
|
998
|
-
|
|
999
|
-
try {
|
|
1000
|
-
// CRITICAL: Activate fill listener EARLY - before ANY operations that place orders
|
|
1001
|
-
// This ensures fills during trigger reset and grid initialization are captured
|
|
1002
|
-
if (typeof this._fillsUnsubscribe === 'function') {
|
|
1003
|
-
await this._fillsUnsubscribe().catch(() => { });
|
|
1004
|
-
}
|
|
1005
|
-
this._fillsUnsubscribe = await chainOrders.listenForFills(this.account || undefined, this._createFillCallback(chainOrders));
|
|
1006
|
-
if (typeof this._fillsUnsubscribe !== 'function') {
|
|
1007
|
-
this._warn('Fill listener did not provide an unsubscribe handler. Shutdown cleanup may be incomplete.');
|
|
1008
|
-
this._fillsUnsubscribe = null;
|
|
1009
|
-
}
|
|
1010
|
-
this._log('Fill listener activated (ready to process fills during startup)');
|
|
1011
|
-
// Register reconnection callback for safety-net sync after websocket reconnect
|
|
1012
|
-
if (!this._reconnectUnregister) {
|
|
1013
|
-
this._reconnectUnregister = registerReconnectHook(() => {
|
|
1014
|
-
this._log('Blockchain connection re-established; scheduling safety-net sync');
|
|
1015
|
-
const runSafetyNetSync = async () => {
|
|
1016
|
-
if (this.manager && this.accountId && !this._shuttingDown && !this.config.dryRun) {
|
|
1017
|
-
// Cap the entire safety-net sync at TIMING.SAFETY_NET_SYNC_TIMEOUT_MS so it can
|
|
1018
|
-
// never hold _fillProcessingLock longer than the
|
|
1019
|
-
// 20s shutdown lock timeout. readOpenOrders +
|
|
1020
|
-
// synchronizeWithChain + batch + persist can be
|
|
1021
|
-
// ~45s worst case without a cap, which would
|
|
1022
|
-
// stall shutdown until the 20s timeout fires.
|
|
1023
|
-
const safetyNetTimeoutMs = this.config.timing?.SAFETY_NET_SYNC_TIMEOUT_MS;
|
|
1024
|
-
let safetyNetTimer;
|
|
1025
|
-
const workPromise = this.manager._fillProcessingLock.acquire(async () => {
|
|
1026
|
-
if (this._shuttingDown)
|
|
1027
|
-
return;
|
|
1028
|
-
const chainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1029
|
-
if (this._shuttingDown)
|
|
1030
|
-
return;
|
|
1031
|
-
const syncResult = await this.manager.synchronizeWithChain(chainOpenOrders, 'readOpenOrders');
|
|
1032
|
-
if (this._shuttingDown)
|
|
1033
|
-
return;
|
|
1034
|
-
if (syncResult?.filledOrders?.length > 0) {
|
|
1035
|
-
this._refreshDynamicWeightDistribution('post-reconnect sync fill');
|
|
1036
|
-
this._log(`Post-reconnect sync: ${syncResult.filledOrders.length} grid order(s) found filled.`, 'info');
|
|
1037
|
-
await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'post-reconnect sync fill');
|
|
1038
|
-
if (this._shuttingDown)
|
|
1039
|
-
return;
|
|
1040
|
-
}
|
|
1041
|
-
this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
|
|
1042
|
-
await this.manager.persistGrid();
|
|
1043
|
-
// Cancel any dust created by reconnect fills immediately.
|
|
1044
|
-
if (!this._shuttingDown) {
|
|
1045
|
-
try {
|
|
1046
|
-
const reconnectHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
1047
|
-
await this._cancelDustOrders({
|
|
1048
|
-
buy: reconnectHealth.buyDustOrders,
|
|
1049
|
-
sell: reconnectHealth.sellDustOrders,
|
|
1050
|
-
});
|
|
1051
|
-
}
|
|
1052
|
-
catch (_dustErr) {
|
|
1053
|
-
this._warn(`[RECONNECT] Dust cancel failed: ${_dustErr.message}`);
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
});
|
|
1057
|
-
try {
|
|
1058
|
-
await Promise.race([
|
|
1059
|
-
workPromise,
|
|
1060
|
-
new Promise((_, reject) => {
|
|
1061
|
-
safetyNetTimer = setTimeout(() => reject(new Error(`Safety-net sync exceeded ${safetyNetTimeoutMs}ms cap`)), safetyNetTimeoutMs);
|
|
1062
|
-
})
|
|
1063
|
-
]);
|
|
1064
|
-
}
|
|
1065
|
-
catch (capErr) {
|
|
1066
|
-
const fallback = await Promise.race([
|
|
1067
|
-
workPromise.then(() => ({ ok: true })),
|
|
1068
|
-
new Promise(resolve => setTimeout(() => resolve({ ok: false }), 0))
|
|
1069
|
-
]);
|
|
1070
|
-
if (fallback.ok) {
|
|
1071
|
-
this._log(`Safety-net sync completed despite timeout — ignoring spurious error.`, 'info');
|
|
1072
|
-
}
|
|
1073
|
-
else {
|
|
1074
|
-
this._warn(`Post-reconnect safety-net sync aborted: ${capErr?.message || capErr}`);
|
|
1075
|
-
}
|
|
1076
|
-
}
|
|
1077
|
-
finally {
|
|
1078
|
-
if (safetyNetTimer)
|
|
1079
|
-
clearTimeout(safetyNetTimer);
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
};
|
|
1083
|
-
// The setImmediate callback returns a Promise; we MUST attach
|
|
1084
|
-
// a .catch so a rejection here does not propagate to
|
|
1085
|
-
// process.on('unhandledRejection') and tear down the bot.
|
|
1086
|
-
setImmediate(() => {
|
|
1087
|
-
runSafetyNetSync().catch(err => {
|
|
1088
|
-
try {
|
|
1089
|
-
this._warn('Post-reconnect safety-net sync failed: ' + (err?.message || err));
|
|
1090
|
-
}
|
|
1091
|
-
catch (_warnErr) {
|
|
1092
|
-
// _warn itself inaccessible — last line of defense.
|
|
1093
|
-
}
|
|
1094
|
-
});
|
|
1095
|
-
});
|
|
1096
|
-
});
|
|
1097
|
-
}
|
|
1098
|
-
// CRITICAL: Handle any pending trigger file reset FIRST before any other startup operations
|
|
1099
|
-
const hadTriggerReset = await this._handlePendingTriggerReset();
|
|
1100
|
-
// CRITICAL: After trigger reset, skip normal startup - grid is already fully initialized
|
|
1101
|
-
// The trigger reset already did: grid init, order placement, sync, and persistence
|
|
1102
|
-
if (hadTriggerReset) {
|
|
1103
|
-
this._log('Trigger reset completed. Skipping normal startup grid initialization.');
|
|
1104
|
-
// Post-bootstrap validation and fill processing
|
|
1105
|
-
await this.manager._fillProcessingLock.acquire(async () => {
|
|
1106
|
-
// STEP 1: Check for fills that occurred during trigger reset
|
|
1107
|
-
// These are orders that got filled while Grid.recalculateGrid() was running.
|
|
1108
|
-
// The filled slots need new orders placed on them.
|
|
1109
|
-
if (this._incomingFillQueue.length > 0) {
|
|
1110
|
-
this._log(`[POST-RESET] ${this._incomingFillQueue.length} fill(s) detected during trigger reset. Processing...`);
|
|
1111
|
-
// Process fills - this will place new orders on the filled slots
|
|
1112
|
-
// Use normal fill processing since bootstrap is complete
|
|
1113
|
-
const fills = this._incomingFillQueue.splice(0);
|
|
1114
|
-
const processedFillKeys = new Set();
|
|
1115
|
-
let requiresOpenOrdersSync = false;
|
|
1116
|
-
for (const fill of fills) {
|
|
1117
|
-
if (!fill || fill.op?.[0] !== 4)
|
|
1118
|
-
continue;
|
|
1119
|
-
const fillOp = fill.op[1];
|
|
1120
|
-
const gridOrder = this.manager.orders.get(fillOp.order_id) ||
|
|
1121
|
-
Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
|
|
1122
|
-
if (!gridOrder) {
|
|
1123
|
-
// CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
|
|
1124
|
-
// This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
|
|
1125
|
-
let orphanFillKey = buildFillKey(fill);
|
|
1126
|
-
if (!orphanFillKey) {
|
|
1127
|
-
orphanFillKey = this._buildOrphanFillFallbackKey(fill);
|
|
1128
|
-
}
|
|
1129
|
-
if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[POST-RESET]', fillOp.order_id)) {
|
|
1130
|
-
continue;
|
|
1131
|
-
}
|
|
1132
|
-
this._log(`[POST-RESET] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
|
|
1133
|
-
const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
|
|
1134
|
-
context: 'POST-RESET',
|
|
1135
|
-
logger: { log: this._log.bind(this) },
|
|
1136
|
-
replayMessage: (op) => `[POST-RESET] Replay detected for orphan fill ${op.order_id}; skipping duplicate credit`
|
|
1137
|
-
});
|
|
1138
|
-
if (accountingResult.status === 'missing_key') {
|
|
1139
|
-
requiresOpenOrdersSync = true;
|
|
1140
|
-
}
|
|
1141
|
-
continue;
|
|
1142
|
-
}
|
|
1143
|
-
this._log(`[POST-RESET] Processing fill for ${gridOrder.type} order ${gridOrder.id} at price ${gridOrder.price}`);
|
|
1144
|
-
const trackedFillKey = buildFillKey(fill);
|
|
1145
|
-
if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[POST-RESET]', fillOp.order_id)) {
|
|
1146
|
-
continue;
|
|
1147
|
-
}
|
|
1148
|
-
this.manager.lockOrders([gridOrder.id]);
|
|
1149
|
-
try {
|
|
1150
|
-
const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
|
|
1151
|
-
context: 'POST-RESET',
|
|
1152
|
-
logger: { log: this._log.bind(this) },
|
|
1153
|
-
replayMessage: (op) => `[POST-RESET] Replay detected for ${op.order_id}; skipping duplicate rebalance`
|
|
1154
|
-
});
|
|
1155
|
-
if (accountingResult.status === 'missing_key') {
|
|
1156
|
-
requiresOpenOrdersSync = true;
|
|
1157
|
-
continue;
|
|
1158
|
-
}
|
|
1159
|
-
if (accountingResult.status !== 'applied') {
|
|
1160
|
-
continue;
|
|
1161
|
-
}
|
|
1162
|
-
// Process this fill through the full rebalance pipeline
|
|
1163
|
-
// This will shift the boundary and place a new order on the filled slot
|
|
1164
|
-
const result = await this._processFillsWithBatching([gridOrder], new Set(), `[POST-RESET] fill ${gridOrder.id}`);
|
|
1165
|
-
if (result.aborted) {
|
|
1166
|
-
this._warn('[POST-RESET] Aborted batch due to illegal state; skipping grid persistence this cycle');
|
|
1167
|
-
continue;
|
|
1168
|
-
}
|
|
1169
|
-
}
|
|
1170
|
-
finally {
|
|
1171
|
-
this.manager.unlockOrders([gridOrder.id]);
|
|
1172
|
-
}
|
|
1173
|
-
}
|
|
1174
|
-
if (requiresOpenOrdersSync) {
|
|
1175
|
-
this._log('[POST-RESET] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
|
|
1176
|
-
const postResetChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1177
|
-
const syncResult = await this.manager.syncFromOpenOrders(postResetChainOpenOrders);
|
|
1178
|
-
if (syncResult.filledOrders?.length > 0) {
|
|
1179
|
-
await this._processFillsWithBatching(syncResult.filledOrders, new Set(), '[POST-RESET] open-orders fallback');
|
|
1180
|
-
}
|
|
1181
|
-
}
|
|
1182
|
-
await this._flushProcessedFillPersistence('post-reset-batch');
|
|
1183
|
-
this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
|
|
1184
|
-
await this.manager.persistGrid();
|
|
1185
|
-
}
|
|
1186
|
-
// STEP 2: Refresh chain truth before spread correction. Trigger
|
|
1187
|
-
// reset can create/cancel orders and fills can arrive while the
|
|
1188
|
-
// reset is running; spread decisions must not use stale local grid.
|
|
1189
|
-
const { aborted: postResetAborted, hasUnmatched: postResetUnmatched } = await this._syncOpenOrdersAndProcessFills('[POST-RESET] pre-spread');
|
|
1190
|
-
if (postResetUnmatched) {
|
|
1191
|
-
this._warn(`[POST-RESET] Skipping spread correction: ${postResetUnmatched} unmatched chain order(s) require maintenance reconciliation`);
|
|
1192
|
-
}
|
|
1193
|
-
// STEP 3: Spread check AFTER fills are processed and chain truth refreshed
|
|
1194
|
-
await this.manager.recalculateFunds();
|
|
1195
|
-
if (!postResetAborted && !postResetUnmatched) {
|
|
1196
|
-
const spreadResult = await this.manager.checkSpreadCondition(BitShares, this.updateOrdersOnChainPlan.bind(this));
|
|
1197
|
-
if (spreadResult && spreadResult.ordersPlaced > 0) {
|
|
1198
|
-
this._log(`✓ Spread correction after trigger reset: ${spreadResult.ordersPlaced} order(s) placed`);
|
|
1199
|
-
await this._persistAndRecoverIfNeeded();
|
|
1200
|
-
}
|
|
1201
|
-
}
|
|
1202
|
-
// Cancel any dust created by post-reset fills immediately.
|
|
1203
|
-
if (!this._shuttingDown) {
|
|
1204
|
-
try {
|
|
1205
|
-
const postResetHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
1206
|
-
await this._cancelDustOrders({
|
|
1207
|
-
buy: postResetHealth.buyDustOrders,
|
|
1208
|
-
sell: postResetHealth.sellDustOrders,
|
|
1209
|
-
});
|
|
1210
|
-
}
|
|
1211
|
-
catch (_dustErr) {
|
|
1212
|
-
this._warn(`[POST-RESET] Dust cancel failed: ${_dustErr.message}`);
|
|
1213
|
-
}
|
|
1214
|
-
}
|
|
1215
|
-
this._log('Bootstrap phase complete - fill processing resumed', 'info');
|
|
1216
|
-
});
|
|
1217
|
-
await this._setupTriggerFileDetection();
|
|
1218
|
-
await this._setupCreditRuntime();
|
|
1219
|
-
await this._refreshAndSyncCreditRuntime();
|
|
1220
|
-
this._setupBlockchainFetchInterval();
|
|
1221
|
-
this._setupCreditWatchdogInterval();
|
|
1222
|
-
this._setupCredentialDaemonWatchdogInterval();
|
|
1223
|
-
this._setupDustHealthCheckInterval();
|
|
1224
|
-
await this._runDustHealthCheck();
|
|
1225
|
-
this._log('[DUST] Startup health check complete');
|
|
1226
|
-
if (this._isOpenOrdersSyncLoopEnabled()) {
|
|
1227
|
-
this._startOpenOrdersSyncLoop();
|
|
1228
|
-
}
|
|
1229
|
-
else {
|
|
1230
|
-
this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
|
|
1231
|
-
}
|
|
1232
|
-
this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
|
|
1233
|
-
return; // Skip normal startup path
|
|
1234
|
-
}
|
|
1235
|
-
// Restore persisted BTS fee
|
|
1236
|
-
// SAFE: Done at startup before orders are created, and within fill lock when needed
|
|
1237
|
-
this.manager.resetFunds();
|
|
1238
|
-
// CRITICAL FIX: Restore BTS fees owed from persistence
|
|
1239
|
-
if (persistedBtsFeesOwed && persistedBtsFeesOwed > 0) {
|
|
1240
|
-
this.manager.funds.btsFeesOwed = Number(persistedBtsFeesOwed);
|
|
1241
|
-
}
|
|
1242
|
-
// Restore BTS balance for non-BTS pairs
|
|
1243
|
-
if (this.config.assetA !== 'BTS' && this.config.assetB !== 'BTS') {
|
|
1244
|
-
if (persistedBtsBalance && typeof persistedBtsBalance === 'object') {
|
|
1245
|
-
this.manager.btsBalance = {
|
|
1246
|
-
free: persistedBtsBalance.free || 0,
|
|
1247
|
-
total: persistedBtsBalance.total || 0,
|
|
1248
|
-
locked: persistedBtsBalance.locked || 0
|
|
1249
|
-
};
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
// Restore recently queued fill keys for crash-durable dedup window.
|
|
1253
|
-
// All persisted keys are restored; the in-memory TTL (_fillDedupeWindowMs)
|
|
1254
|
-
// will naturally evict any that are already expired on the next fill cycle.
|
|
1255
|
-
if (persistedRecentFillKeys && typeof persistedRecentFillKeys === 'object') {
|
|
1256
|
-
for (const [fillKey, timestamp] of Object.entries(persistedRecentFillKeys)) {
|
|
1257
|
-
this._recentlyQueuedFills.set(fillKey, Number(timestamp));
|
|
1258
|
-
}
|
|
1259
|
-
this._log(`Restored ${Object.keys(persistedRecentFillKeys).length} recently queued fill key(s) from persisted snapshot`, 'debug');
|
|
1260
|
-
}
|
|
1261
|
-
if (!this.config.dryRun && !this.accountId) {
|
|
1262
|
-
throw new Error('Cannot start bot without a resolved account ID');
|
|
1263
|
-
}
|
|
1264
|
-
// Use this.accountId which was set during initialize()
|
|
1265
|
-
const chainOpenOrders = this.config.dryRun ? [] : await chainOrders.readOpenOrders(this.accountId);
|
|
1266
|
-
let shouldRegenerate = false;
|
|
1267
|
-
if (!persistedGrid || persistedGrid.length === 0) {
|
|
1268
|
-
shouldRegenerate = true;
|
|
1269
|
-
this._log('No persisted grid found. Generating new grid.');
|
|
1270
|
-
}
|
|
1271
|
-
else {
|
|
1272
|
-
await this.manager._initializeAssets();
|
|
1273
|
-
const decision = await decideStartupGridAction({
|
|
1274
|
-
persistedGrid,
|
|
1275
|
-
chainOpenOrders,
|
|
1276
|
-
manager: this.manager,
|
|
1277
|
-
logger: { log: (msg) => this._log(msg) },
|
|
1278
|
-
storeGrid: async (orders) => {
|
|
1279
|
-
// Pass the snapshot directly to persistGrid so the live
|
|
1280
|
-
// `manager.orders` map is not briefly swapped (which would
|
|
1281
|
-
// leave the _ordersByState/_ordersByType indexes inconsistent
|
|
1282
|
-
// with the map for the duration of the persist call).
|
|
1283
|
-
await this.manager.persistGrid(orders);
|
|
1284
|
-
},
|
|
1285
|
-
attemptResumeFn: attemptResumePersistedGridByPriceMatch,
|
|
1286
|
-
});
|
|
1287
|
-
shouldRegenerate = decision.shouldRegenerate;
|
|
1288
|
-
if (shouldRegenerate && chainOpenOrders.length === 0) {
|
|
1289
|
-
this._log('Persisted grid found, but no matching active orders on-chain. Generating new grid.');
|
|
1290
|
-
}
|
|
1291
|
-
// Also log when regeneration is needed but other pairs' orders
|
|
1292
|
-
// suppress the "no matching orders" message above.
|
|
1293
|
-
if (shouldRegenerate && chainOpenOrders.length > 0 && this.manager?.assets) {
|
|
1294
|
-
const orderCount = chainOpenOrders.filter(o => parseChainOrder(o, this.manager.assets) !== null).length;
|
|
1295
|
-
if (orderCount === 0) {
|
|
1296
|
-
this._log(`Persisted grid found with no matching orders (${chainOpenOrders.length} other-pair order(s) on account). Generating new grid.`);
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
// Restore BTS fees owed ONLY if we're NOT regenerating the grid
|
|
1301
|
-
if (!shouldRegenerate) {
|
|
1302
|
-
// CRITICAL: Restore BTS fees owed from blockchain operations
|
|
1303
|
-
if (persistedBtsFeesOwed > 0) {
|
|
1304
|
-
this.manager.funds.btsFeesOwed = persistedBtsFeesOwed;
|
|
1305
|
-
this._log(`✓ Restored BTS fees owed: ${Format.formatAmount8(persistedBtsFeesOwed)} BTS`);
|
|
1306
|
-
}
|
|
1307
|
-
}
|
|
1308
|
-
else {
|
|
1309
|
-
this._log(`ℹ Grid regenerating - resetting BTS fees to clean state`);
|
|
1310
|
-
this.manager.funds.btsFeesOwed = 0;
|
|
1311
|
-
}
|
|
1312
|
-
// CRITICAL: Use fill lock during ENTIRE startup synchronization to prevent races.
|
|
1313
|
-
// This includes grid init, finishBootstrap, and maintenance - all in one atomic block.
|
|
1314
|
-
// Lock order: _fillProcessingLock → _divergenceLock (canonical order, same as _consumeFillQueue)
|
|
1315
|
-
await this.manager._fillProcessingLock.acquire(async () => {
|
|
1316
|
-
try {
|
|
1317
|
-
this._refreshDynamicWeightDistribution('startup');
|
|
1318
|
-
if (shouldRegenerate) {
|
|
1319
|
-
await this.manager._initializeAssets();
|
|
1320
|
-
if (Array.isArray(chainOpenOrders) && chainOpenOrders.length > 0) {
|
|
1321
|
-
this._log('Generating new grid and syncing with existing on-chain orders...');
|
|
1322
|
-
await Grid.initializeGrid(this.manager);
|
|
1323
|
-
await this.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
|
|
1324
|
-
const rebalanceResult = await reconcileGridOrders({
|
|
1325
|
-
manager: this.manager,
|
|
1326
|
-
config: this.config,
|
|
1327
|
-
account: this.account,
|
|
1328
|
-
privateKey: this.privateKey,
|
|
1329
|
-
chainOrders,
|
|
1330
|
-
chainOpenOrders,
|
|
1331
|
-
});
|
|
1332
|
-
await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
|
|
1333
|
-
}
|
|
1334
|
-
else {
|
|
1335
|
-
this._log('Generating new grid and placing initial orders on-chain...');
|
|
1336
|
-
await this.placeInitialOrders();
|
|
1337
|
-
}
|
|
1338
|
-
await this._persistAndRecoverIfNeeded();
|
|
1339
|
-
}
|
|
1340
|
-
else {
|
|
1341
|
-
this._log('Found active session. Loading and syncing existing grid.');
|
|
1342
|
-
await Grid.loadGrid(this.manager, persistedGrid, persistedBoundaryIdx);
|
|
1343
|
-
let startupChainOpenOrders = chainOpenOrders;
|
|
1344
|
-
const syncResult = await this.manager.syncFromOpenOrders(startupChainOpenOrders, { skipAccounting: true });
|
|
1345
|
-
// Process price corrections queued during startup sync.
|
|
1346
|
-
// These are not picked up by _consumeFillQueue until a fill
|
|
1347
|
-
// arrives, which may never come for an idle market.
|
|
1348
|
-
if (syncResult.ordersNeedingCorrection?.length > 0) {
|
|
1349
|
-
await correctAllPriceMismatches(this.manager, this.account, this.privateKey, chainOrders);
|
|
1350
|
-
}
|
|
1351
|
-
if (syncResult.filledOrders && syncResult.filledOrders.length > 0) {
|
|
1352
|
-
this._log(`Startup sync: ${syncResult.filledOrders.length} grid order(s) found filled. Processing proceeds.`, 'info');
|
|
1353
|
-
const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'startup sync fill rebalance', { skipAccountTotalsUpdate: true });
|
|
1354
|
-
if (!batchResult?.aborted) {
|
|
1355
|
-
// Refresh open orders so startup reconcile works with post-batch chain reality
|
|
1356
|
-
// and avoids reconciling against a stale pre-batch snapshot.
|
|
1357
|
-
startupChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1358
|
-
await this.manager.synchronizeWithChain(startupChainOpenOrders, 'readOpenOrders');
|
|
1359
|
-
}
|
|
1360
|
-
}
|
|
1361
|
-
const rebalanceResult = await reconcileGridOrders({
|
|
1362
|
-
manager: this.manager,
|
|
1363
|
-
config: this.config,
|
|
1364
|
-
account: this.account,
|
|
1365
|
-
privateKey: this.privateKey,
|
|
1366
|
-
chainOrders,
|
|
1367
|
-
chainOpenOrders: startupChainOpenOrders,
|
|
1368
|
-
});
|
|
1369
|
-
await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (loaded grid)');
|
|
1370
|
-
// Dust state is no longer persisted — cancelled immediately on detection.
|
|
1371
|
-
await this._persistAndRecoverIfNeeded();
|
|
1372
|
-
// Gap 1: Grid snapshot sanity check — shared logic with recovery path.
|
|
1373
|
-
await this._rejectCorruptedGridSnapshot('startup');
|
|
1374
|
-
}
|
|
1375
|
-
// Drain any fills that arrived during startup while still in bootstrap
|
|
1376
|
-
// mode. Safe to call directly since we already hold _fillProcessingLock
|
|
1377
|
-
// and _processFillsWithBootstrapMode does NOT re-acquire it.
|
|
1378
|
-
if (this._incomingFillQueue.length > 0) {
|
|
1379
|
-
this._log(`[STARTUP] Processing ${this._incomingFillQueue.length} queued fill(s) before bootstrap ends`);
|
|
1380
|
-
await this._processFillsWithBootstrapMode(chainOrders);
|
|
1381
|
-
}
|
|
1382
|
-
this.manager.finishBootstrap();
|
|
1383
|
-
// Refresh account totals after bootstrap to eliminate the timing
|
|
1384
|
-
// gap between the initial balance fetch and grid operations (sync,
|
|
1385
|
-
// reconcile, fills). Without this, the first maintenance cycle sees
|
|
1386
|
-
// a fund drift (expected at bootstrap) and triggers an unnecessary
|
|
1387
|
-
// invariant violation + full recovery cycle.
|
|
1388
|
-
// Bound by a timeout to avoid blocking _fillProcessingLock on a
|
|
1389
|
-
// flaky node. If the fetch times out, continue with cached values;
|
|
1390
|
-
// the next periodic maintenance cycle will retry.
|
|
1391
|
-
const FETCH_TIMEOUT_MS = 30000;
|
|
1392
|
-
let _fetchTimeoutHandle;
|
|
1393
|
-
try {
|
|
1394
|
-
await Promise.race([
|
|
1395
|
-
this.manager.fetchAccountTotals(),
|
|
1396
|
-
new Promise((_, reject) => {
|
|
1397
|
-
_fetchTimeoutHandle = setTimeout(() => reject(new Error('timeout')), FETCH_TIMEOUT_MS);
|
|
1398
|
-
})
|
|
1399
|
-
]);
|
|
1400
|
-
}
|
|
1401
|
-
catch (fetchErr) {
|
|
1402
|
-
this._log(`[STARTUP] [${this.config?.botKey || 'unknown'}] fetchAccountTotals ${fetchErr.message === 'timeout' ? 'timed out' : 'failed'} (${fetchErr.message}). Continuing with cached account totals.`, 'warn');
|
|
1403
|
-
}
|
|
1404
|
-
finally {
|
|
1405
|
-
clearTimeout(_fetchTimeoutHandle);
|
|
1406
|
-
}
|
|
1407
|
-
// Perform initial grid maintenance (thresholds, divergence, spread, health)
|
|
1408
|
-
// Consolidated into shared logic to ensure consistent behavior at boot and runtime.
|
|
1409
|
-
// CRITICAL: Pass lockAlreadyHeld since we're inside _fillProcessingLock.acquire()
|
|
1410
|
-
await this._runGridMaintenance('startup');
|
|
1411
|
-
// Cancel any dust from a previous bot lifetime immediately.
|
|
1412
|
-
const startupHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
1413
|
-
await this._cancelDustOrders({
|
|
1414
|
-
buy: startupHealth.buyDustOrders,
|
|
1415
|
-
sell: startupHealth.sellDustOrders,
|
|
1416
|
-
});
|
|
1417
|
-
this._log('Bootstrap phase complete - fill processing resumed', 'info');
|
|
1418
|
-
}
|
|
1419
|
-
finally {
|
|
1420
|
-
// CRITICAL: Always clear bootstrap flag, even on error
|
|
1421
|
-
this.manager.finishBootstrap();
|
|
1422
|
-
}
|
|
1423
|
-
});
|
|
1424
|
-
await this._setupTriggerFileDetection();
|
|
1425
|
-
await this._setupCreditRuntime();
|
|
1426
|
-
await this._refreshAndSyncCreditRuntime();
|
|
1427
|
-
await this._runCreditRuntimeMaintenance('startup');
|
|
1428
|
-
this._setupBlockchainFetchInterval();
|
|
1429
|
-
this._setupCreditWatchdogInterval();
|
|
1430
|
-
this._setupCredentialDaemonWatchdogInterval();
|
|
1431
|
-
// Periodic dust health check — catches partials that fell below threshold
|
|
1432
|
-
// without triggering the post-fill pipeline. Cancels immediately inside the
|
|
1433
|
-
// fill-processing lock to avoid racing with fill batches or sync operations.
|
|
1434
|
-
this._setupDustHealthCheckInterval();
|
|
1435
|
-
await this._runDustHealthCheck();
|
|
1436
|
-
this._log('[DUST] Startup health check complete');
|
|
1437
|
-
if (this._isOpenOrdersSyncLoopEnabled()) {
|
|
1438
|
-
this._startOpenOrdersSyncLoop();
|
|
1439
|
-
}
|
|
1440
|
-
else {
|
|
1441
|
-
this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
|
|
1442
|
-
}
|
|
1443
|
-
this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
|
|
1444
|
-
}
|
|
1445
|
-
catch (err) {
|
|
1446
|
-
this._warn(`Error during grid initialization: ${err.message}`);
|
|
1447
|
-
await this.shutdown();
|
|
1448
|
-
throw err;
|
|
1449
|
-
}
|
|
595
|
+
return DexbotStartupRuntime.finishStartupSequence(this, startupState);
|
|
1450
596
|
}
|
|
1451
597
|
/**
|
|
1452
598
|
* Create the fill callback for listenForFills.
|
|
@@ -1465,35 +611,10 @@ class DEXBot {
|
|
|
1465
611
|
* @returns {Promise<{syncResult: Object|null, aborted: boolean, hasUnmatched: number, openOrders: Array|null}>}
|
|
1466
612
|
*/
|
|
1467
613
|
async _syncOpenOrdersAndProcessFills(tag) {
|
|
1468
|
-
|
|
1469
|
-
return { syncResult: null, aborted: false, hasUnmatched: 0, openOrders: null };
|
|
1470
|
-
}
|
|
1471
|
-
try {
|
|
1472
|
-
let openOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1473
|
-
const syncResult = await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
|
|
1474
|
-
let aborted = false;
|
|
1475
|
-
if (syncResult?.filledOrders?.length > 0) {
|
|
1476
|
-
this._refreshDynamicWeightDistribution(`${tag} sync-fill`);
|
|
1477
|
-
this._log(`[SYNC-CHAIN] ${syncResult.filledOrders.length} filled order(s) found during ${tag}`, 'info');
|
|
1478
|
-
const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), `${tag} sync-fill`);
|
|
1479
|
-
if (!batchResult?.aborted) {
|
|
1480
|
-
openOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
1481
|
-
await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
|
|
1482
|
-
}
|
|
1483
|
-
else {
|
|
1484
|
-
aborted = true;
|
|
1485
|
-
}
|
|
1486
|
-
}
|
|
1487
|
-
const hasUnmatched = syncResult?.unmatchedChainOrders?.length || 0;
|
|
1488
|
-
return { syncResult, aborted, hasUnmatched, openOrders };
|
|
1489
|
-
}
|
|
1490
|
-
catch (err) {
|
|
1491
|
-
this._warn(`[SYNC-CHAIN] Open-orders sync failed during ${tag}: ${err.message}`);
|
|
1492
|
-
return { syncResult: null, aborted: true, hasUnmatched: -1, openOrders: null };
|
|
1493
|
-
}
|
|
614
|
+
return DexbotMaintenanceRuntime.syncOpenOrdersAndProcessFills(this, tag);
|
|
1494
615
|
}
|
|
1495
616
|
_maxConsecutiveFillConsumerFailures() {
|
|
1496
|
-
return this
|
|
617
|
+
return DexbotFillRuntime.maxConsecutiveFillConsumerFailures(this);
|
|
1497
618
|
}
|
|
1498
619
|
/**
|
|
1499
620
|
* Compute the backoff delay for fill-consumer retries after the failure
|
|
@@ -1505,80 +626,10 @@ class DEXBot {
|
|
|
1505
626
|
* @private
|
|
1506
627
|
*/
|
|
1507
628
|
_computeFillConsumerBackoffMs(failures) {
|
|
1508
|
-
|
|
1509
|
-
const initial = fp.CONSUMER_BACKOFF_INITIAL_MS;
|
|
1510
|
-
const max = fp.CONSUMER_BACKOFF_MAX_MS;
|
|
1511
|
-
const stepAfterMax = Math.max(0, failures - this._maxConsecutiveFillConsumerFailures());
|
|
1512
|
-
// 0 -> initial, 1 -> 2*initial, 2 -> 4*initial, ... capped at max.
|
|
1513
|
-
return Math.min(max, initial * Math.pow(2, stepAfterMax));
|
|
629
|
+
return DexbotFillRuntime.computeFillConsumerBackoffMs(this, failures);
|
|
1514
630
|
}
|
|
1515
631
|
_scheduleFillConsumerRestart(chainOrders) {
|
|
1516
|
-
|
|
1517
|
-
if (failures >= this._maxConsecutiveFillConsumerFailures()) {
|
|
1518
|
-
// Past the failure budget: switch from tight setImmediate loop to
|
|
1519
|
-
// exponential backoff. The consumer continues to retry — a transient
|
|
1520
|
-
// outage (e.g., credential daemon recovery) will resume normal
|
|
1521
|
-
// operation as soon as one cycle succeeds and resets the counter.
|
|
1522
|
-
// A permanent failure mode yields slower-but-still-progressing
|
|
1523
|
-
// retries capped at CONSUMER_BACKOFF_MAX_MS, with no permanent
|
|
1524
|
-
// stop that would require a bot restart.
|
|
1525
|
-
const backoffMs = this._computeFillConsumerBackoffMs(failures);
|
|
1526
|
-
const elapsedSec = this._consumeFailureFirstAt
|
|
1527
|
-
? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
|
|
1528
|
-
: null;
|
|
1529
|
-
const elapsed = elapsedSec !== null ? `${elapsedSec}s` : 'unknown';
|
|
1530
|
-
// Escalate log level on sustained failure so operators monitoring
|
|
1531
|
-
// for error/critical alerts are not blind to a stuck consumer.
|
|
1532
|
-
// - warn : within the first escalation window (5-9 failures,
|
|
1533
|
-
// <5 min) — could be a slow recovery
|
|
1534
|
-
// - error : 10+ failures OR 5+ minutes of sustained failure
|
|
1535
|
-
// - critical: 20+ failures OR 15+ minutes of sustained failure —
|
|
1536
|
-
// this is the "permanent fault" signal
|
|
1537
|
-
const sustainedLevel = (failures >= 20 || (elapsedSec !== null && elapsedSec >= 900))
|
|
1538
|
-
? 'critical'
|
|
1539
|
-
: (failures >= 10 || (elapsedSec !== null && elapsedSec >= 300))
|
|
1540
|
-
? 'error'
|
|
1541
|
-
: 'warn';
|
|
1542
|
-
this._log(`[FILL-QUEUE] Fill consumer has failed ${failures} consecutive times over ${elapsed}; ` +
|
|
1543
|
-
`backing off ${Math.round(backoffMs / TIMING.MILLISECONDS_PER_SECOND)}s before retry. ` +
|
|
1544
|
-
`Queue: ${this._incomingFillQueue.length} fills.`, sustainedLevel);
|
|
1545
|
-
setTimeout(() => {
|
|
1546
|
-
if (this._shuttingDown)
|
|
1547
|
-
return;
|
|
1548
|
-
this._consumeFillQueue(chainOrders).catch(err => {
|
|
1549
|
-
if (!this._consumeFailureFirstAt) {
|
|
1550
|
-
this._consumeFailureFirstAt = Date.now();
|
|
1551
|
-
}
|
|
1552
|
-
this._consecutiveConsumeFailures++;
|
|
1553
|
-
const newFailures = this._consecutiveConsumeFailures;
|
|
1554
|
-
const newElapsedSec = this._consumeFailureFirstAt
|
|
1555
|
-
? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
|
|
1556
|
-
: null;
|
|
1557
|
-
const resumeLevel = (newFailures >= 20 || (newElapsedSec !== null && newElapsedSec >= 900))
|
|
1558
|
-
? 'critical'
|
|
1559
|
-
: (newFailures >= 10 || (newElapsedSec !== null && newElapsedSec >= 300))
|
|
1560
|
-
? 'error'
|
|
1561
|
-
: 'warn';
|
|
1562
|
-
this._log(`Fill consumer resume after backoff failed ` +
|
|
1563
|
-
`(${newFailures} total, ` +
|
|
1564
|
-
`next backoff ${Math.round(this._computeFillConsumerBackoffMs(newFailures) / TIMING.MILLISECONDS_PER_SECOND)}s): ` +
|
|
1565
|
-
`${err.message}`, resumeLevel);
|
|
1566
|
-
// Continue the backoff loop. The success path of
|
|
1567
|
-
// _consumeFillQueue resets the counter, breaking the cycle.
|
|
1568
|
-
this._scheduleFillConsumerRestart(chainOrders);
|
|
1569
|
-
});
|
|
1570
|
-
}, backoffMs);
|
|
1571
|
-
return;
|
|
1572
|
-
}
|
|
1573
|
-
setImmediate(() => this._consumeFillQueue(chainOrders).catch(err => {
|
|
1574
|
-
if (!this._consumeFailureFirstAt) {
|
|
1575
|
-
this._consumeFailureFirstAt = Date.now();
|
|
1576
|
-
}
|
|
1577
|
-
this._consecutiveConsumeFailures++;
|
|
1578
|
-
const remaining = this._maxConsecutiveFillConsumerFailures() - this._consecutiveConsumeFailures;
|
|
1579
|
-
this._log(`Fill consumer failed (${this._consecutiveConsumeFailures}/${this._maxConsecutiveFillConsumerFailures()}, ` +
|
|
1580
|
-
`${remaining} attempts remaining): ${err.message}`, this._consecutiveConsumeFailures >= 3 ? 'warn' : 'error');
|
|
1581
|
-
}));
|
|
632
|
+
DexbotFillRuntime.scheduleFillConsumerRestart(this, chainOrders);
|
|
1582
633
|
}
|
|
1583
634
|
/**
|
|
1584
635
|
* Consume queued fills from incomingFillQueue and rebalance.
|
|
@@ -1594,564 +645,7 @@ class DEXBot {
|
|
|
1594
645
|
* @private
|
|
1595
646
|
*/
|
|
1596
647
|
async _consumeFillQueue(chainOrders) {
|
|
1597
|
-
|
|
1598
|
-
// The counter only tracks actual failures, so any healthy deferral
|
|
1599
|
-
// path should also reset the counter. Without this, a sequence of
|
|
1600
|
-
// F-S-F-S-F-S-F-S-F (fail, succeed, fail, succeed, ...) would still
|
|
1601
|
-
// reach the max and trip backoff, OR a failure followed by an empty
|
|
1602
|
-
// queue / shutdown / in-flight batch would leave the counter sticky
|
|
1603
|
-
// and trigger backoff one step sooner on the next real failure.
|
|
1604
|
-
const resetFailureWatchdogIfSet = () => {
|
|
1605
|
-
if (this._consecutiveConsumeFailures > 0 || this._consumeFailureFirstAt > 0) {
|
|
1606
|
-
this._consecutiveConsumeFailures = 0;
|
|
1607
|
-
this._consumeFailureFirstAt = 0;
|
|
1608
|
-
}
|
|
1609
|
-
};
|
|
1610
|
-
// ATOMIC: Only attempt lock acquisition if queue has work
|
|
1611
|
-
// This prevents unnecessary lock contention on empty queues
|
|
1612
|
-
if (this._incomingFillQueue.length === 0) {
|
|
1613
|
-
// Empty queue = consumer is healthy, just idle.
|
|
1614
|
-
resetFailureWatchdogIfSet();
|
|
1615
|
-
return;
|
|
1616
|
-
}
|
|
1617
|
-
// Check shutdown state
|
|
1618
|
-
if (this._shuttingDown) {
|
|
1619
|
-
this._warn('Fill processing skipped: shutdown in progress');
|
|
1620
|
-
resetFailureWatchdogIfSet();
|
|
1621
|
-
return;
|
|
1622
|
-
}
|
|
1623
|
-
if (this._batchInFlight || this._recoverySyncInFlight) {
|
|
1624
|
-
this.manager?.logger?.log?.(`Fill processing deferred: order pipeline active (${this._incomingFillQueue.length} queued)`, 'debug');
|
|
1625
|
-
// A batch is in flight, not a failure. The next iteration will
|
|
1626
|
-
// either succeed (and reset the counter on the success path) or
|
|
1627
|
-
// fail and increment it. Either way, leaving an old failure
|
|
1628
|
-
// count here would double-count.
|
|
1629
|
-
resetFailureWatchdogIfSet();
|
|
1630
|
-
return;
|
|
1631
|
-
}
|
|
1632
|
-
let pendingFillKeysForCurrentCycle = new Set();
|
|
1633
|
-
try {
|
|
1634
|
-
// BOOTSTRAP OPTIMIZATION: During bootstrap, prioritize fill processing over grid-wide checks
|
|
1635
|
-
// Process fills immediately with side-only rebalancing (no expensive full grid recalculations)
|
|
1636
|
-
if (this.manager.isBootstrapping()) {
|
|
1637
|
-
// During bootstrap: skip lock contention checks, process fills directly
|
|
1638
|
-
let bootstrapSkipped = false;
|
|
1639
|
-
await this.manager._fillProcessingLock.acquire(async () => {
|
|
1640
|
-
if (!this.manager.isBootstrapping()) {
|
|
1641
|
-
// Bootstrap finished while waiting for the lock — no
|
|
1642
|
-
// work to do, but the iteration is still healthy.
|
|
1643
|
-
bootstrapSkipped = true;
|
|
1644
|
-
return;
|
|
1645
|
-
}
|
|
1646
|
-
await this._processFillsWithBootstrapMode(chainOrders);
|
|
1647
|
-
});
|
|
1648
|
-
if (bootstrapSkipped) {
|
|
1649
|
-
// Bootstrap-mode lock callback returned without doing
|
|
1650
|
-
// work; the .acquire() success path at line ~1941 that
|
|
1651
|
-
// would normally reset the counter is not reached in
|
|
1652
|
-
// this branch. Reset here so a stale counter from a
|
|
1653
|
-
// prior failure doesn't carry over.
|
|
1654
|
-
resetFailureWatchdogIfSet();
|
|
1655
|
-
}
|
|
1656
|
-
return;
|
|
1657
|
-
}
|
|
1658
|
-
// NORMAL MODE: Non-blocking check if lock already has waiters
|
|
1659
|
-
// This prevents unbounded queue growth while still ensuring processing
|
|
1660
|
-
// Note: We DO proceed if lock is held but has no waiters - we'll wait our turn
|
|
1661
|
-
if (this.manager._fillProcessingLock.getQueueLength() > 0) {
|
|
1662
|
-
this._metrics.lockContentionEvents++;
|
|
1663
|
-
// Deferral, not a failure. Reset the watchdog so the next
|
|
1664
|
-
// call (which may now find an empty queue, or process
|
|
1665
|
-
// successfully) doesn't inherit a stale counter.
|
|
1666
|
-
resetFailureWatchdogIfSet();
|
|
1667
|
-
return;
|
|
1668
|
-
}
|
|
1669
|
-
await this.manager._fillProcessingLock.acquire(async () => {
|
|
1670
|
-
// Reset orphan-fill credit timestamp at the start of each
|
|
1671
|
-
// fill cycle. It is re-set when orphan fills are credited,
|
|
1672
|
-
// and consumed (set to null) on the next fund-invariant check
|
|
1673
|
-
// in accounting.ts, which widens tolerance by 5x while set.
|
|
1674
|
-
// Written to this.manager (not this) because accounting.ts
|
|
1675
|
-
// reads from the OrderManager reference (mgr).
|
|
1676
|
-
// Also cleared by _performStateRecovery after a fresh chain
|
|
1677
|
-
// fetch. The timestamp value itself is not compared against a
|
|
1678
|
-
// window — it acts as a consume-on-read boolean.
|
|
1679
|
-
this.manager._orphanFillsCreditedAt = null;
|
|
1680
|
-
while (this._incomingFillQueue.length > 0) {
|
|
1681
|
-
const batchStartTime = Date.now();
|
|
1682
|
-
// Track max queue depth
|
|
1683
|
-
this._metrics.maxQueueDepth = Math.max(this._metrics.maxQueueDepth, this._incomingFillQueue.length);
|
|
1684
|
-
// 1. Take snapshot of current work (ATOMIC: splice removes and returns fills atomically)
|
|
1685
|
-
const allFills = this._incomingFillQueue.splice(0); // Atomically clear and get all fills
|
|
1686
|
-
const validFills = [];
|
|
1687
|
-
const processedFillKeys = new Set();
|
|
1688
|
-
pendingFillKeysForCurrentCycle = new Set();
|
|
1689
|
-
let requiresOpenOrdersSync = false;
|
|
1690
|
-
// 2. Filter and Deduplicate (Standard Logic)
|
|
1691
|
-
for (const fill of allFills) {
|
|
1692
|
-
if (fill && fill.op && fill.op[0] === FILL_PROCESSING.OPERATION_TYPE) {
|
|
1693
|
-
const fillOp = fill.op[1];
|
|
1694
|
-
// SELF-CANCEL GUARD: only drop malformed, cancel-like
|
|
1695
|
-
// artifacts for an order the local process just cancelled.
|
|
1696
|
-
// Real fill_order ops carry economic data and must still be
|
|
1697
|
-
// accounted even if they arrive shortly after a successful
|
|
1698
|
-
// cancel broadcast.
|
|
1699
|
-
const hasFillEconomics = fillOp?.pays?.asset_id && fillOp?.pays?.amount != null
|
|
1700
|
-
&& fillOp?.receives?.asset_id && fillOp?.receives?.amount != null;
|
|
1701
|
-
if (chainOrders && typeof chainOrders.wasRecentlyOwnCancelled === 'function'
|
|
1702
|
-
&& chainOrders.wasRecentlyOwnCancelled(fillOp.order_id)
|
|
1703
|
-
&& !hasFillEconomics) {
|
|
1704
|
-
this.manager.logger.log(`[SELF-CANCEL] Skipping non-economic fill artifact for order ${fillOp.order_id} (just cancelled by this bot)`, 'debug');
|
|
1705
|
-
continue;
|
|
1706
|
-
}
|
|
1707
|
-
// ACCOUNT VALIDATION: Verify the filled order belongs to this bot's account/grid
|
|
1708
|
-
// Only process fills for orders we actually manage
|
|
1709
|
-
const gridOrder = this.manager.orders.get(fillOp.order_id) ||
|
|
1710
|
-
Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
|
|
1711
|
-
if (!gridOrder) {
|
|
1712
|
-
// Check if this order was already freed by stale-order batch cleanup.
|
|
1713
|
-
// When a batch fails due to a stale order reference, the cleanup converts the
|
|
1714
|
-
// slot to VIRTUAL/SPREAD, releasing committed funds to chainFree. If we also
|
|
1715
|
-
// credit the fill proceeds here, we double-count the capital.
|
|
1716
|
-
const staleMarkedAt = this._staleCleanedOrderIds.get(fillOp.order_id);
|
|
1717
|
-
if (staleMarkedAt != null) {
|
|
1718
|
-
const staleAgeMs = Date.now() - staleMarkedAt;
|
|
1719
|
-
if (staleAgeMs <= this._staleCleanupRetentionMs) {
|
|
1720
|
-
this.manager.logger.log(`[ORPHAN-FILL] Skipping double-credit for stale-cleaned order ${fillOp.order_id} ` +
|
|
1721
|
-
`(funds already freed by batch cleanup, age=${staleAgeMs}ms)`, 'warn');
|
|
1722
|
-
continue;
|
|
1723
|
-
}
|
|
1724
|
-
this._staleCleanedOrderIds.delete(fillOp.order_id);
|
|
1725
|
-
}
|
|
1726
|
-
// Legitimate orphan fill: order was virtualized during sequential processing
|
|
1727
|
-
// but a fill arrived afterward. Credit proceeds to maintain fund tracking.
|
|
1728
|
-
let orphanFillKey = buildFillKey(fill);
|
|
1729
|
-
if (!orphanFillKey) {
|
|
1730
|
-
orphanFillKey = this._buildOrphanFillFallbackKey(fill);
|
|
1731
|
-
}
|
|
1732
|
-
if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[ORPHAN-FILL]', fillOp.order_id)) {
|
|
1733
|
-
continue;
|
|
1734
|
-
}
|
|
1735
|
-
this.manager.logger.log(`[ORPHAN-FILL] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
|
|
1736
|
-
const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
|
|
1737
|
-
context: 'ORPHAN-FILL',
|
|
1738
|
-
replayMessage: (op) => `[ORPHAN-FILL] Replay detected for ${op.order_id}; skipping duplicate credit`
|
|
1739
|
-
});
|
|
1740
|
-
if (accountingResult.status === 'missing_key') {
|
|
1741
|
-
requiresOpenOrdersSync = true;
|
|
1742
|
-
}
|
|
1743
|
-
// Record orphan fill credit timestamp for fund invariant
|
|
1744
|
-
// tolerance widening. Written to this.manager because
|
|
1745
|
-
// accounting.ts reads from the OrderManager (mgr).
|
|
1746
|
-
this.manager._orphanFillsCreditedAt = Date.now();
|
|
1747
|
-
// Don't add to validFills - we can't do rebalancing without a grid slot
|
|
1748
|
-
// But the funds are now credited, preventing fund invariant violation
|
|
1749
|
-
continue;
|
|
1750
|
-
}
|
|
1751
|
-
// Process both maker and taker fills for our grid orders
|
|
1752
|
-
// Grid validation ensures we only process fills belonging to our account
|
|
1753
|
-
// Taker fills are included because the bot may execute market orders or act as taker
|
|
1754
|
-
const roleStr = fillOp.is_maker !== false ? 'maker' : 'taker';
|
|
1755
|
-
this.manager.logger.log(`Processing ${roleStr} fill for order ${fillOp.order_id}`, 'debug');
|
|
1756
|
-
const fillKey = buildFillKey(fill);
|
|
1757
|
-
if (!fillKey) {
|
|
1758
|
-
this.manager.logger.log(`[FILL] Missing history id for order ${fillOp.order_id} block ${fill.block_num}; deferring to open-orders sync`, 'warn');
|
|
1759
|
-
requiresOpenOrdersSync = true;
|
|
1760
|
-
continue;
|
|
1761
|
-
}
|
|
1762
|
-
if (!this._isNewFillKey(fillKey, processedFillKeys, '[FILL]', fillOp.order_id)) {
|
|
1763
|
-
continue;
|
|
1764
|
-
}
|
|
1765
|
-
validFills.push(fill);
|
|
1766
|
-
// Log info
|
|
1767
|
-
const paysAmount = fillOp.pays ? fillOp.pays.amount : '?';
|
|
1768
|
-
const receivesAmount = fillOp.receives ? fillOp.receives.amount : '?';
|
|
1769
|
-
this._log(`\n===== FILL DETECTED =====`);
|
|
1770
|
-
this._log(`Order ID: ${fillOp.order_id}`);
|
|
1771
|
-
this._log(`Pays: ${paysAmount}, Receives: ${receivesAmount}`);
|
|
1772
|
-
this._log(`Block: ${fill.block_num} (History ID: ${fill.id || 'N/A'})`);
|
|
1773
|
-
this._log(`=========================\n`);
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1776
|
-
// Clean up short-lived queue dedupe cache to prevent memory leak.
|
|
1777
|
-
const cleanupTimestamp = Date.now();
|
|
1778
|
-
let cleanedCount = 0;
|
|
1779
|
-
for (const [key, timestamp] of this._recentlyQueuedFills) {
|
|
1780
|
-
if (cleanupTimestamp - timestamp > this._fillDedupeWindowMs) {
|
|
1781
|
-
this._recentlyQueuedFills.delete(key);
|
|
1782
|
-
cleanedCount++;
|
|
1783
|
-
}
|
|
1784
|
-
}
|
|
1785
|
-
if (cleanedCount > 0) {
|
|
1786
|
-
this.manager.logger.log(`Cleaned ${cleanedCount} old queued fill records. Remaining: ${this._recentlyQueuedFills.size}`, 'debug');
|
|
1787
|
-
}
|
|
1788
|
-
if (validFills.length === 0 && !requiresOpenOrdersSync)
|
|
1789
|
-
continue; // Loop back for more
|
|
1790
|
-
// 3. Sync and Collect Filled Orders
|
|
1791
|
-
let allFilledOrders = [];
|
|
1792
|
-
let ordersNeedingCorrection = [];
|
|
1793
|
-
const pendingGhostOrders = new Set();
|
|
1794
|
-
const fillMode = chainOrders.getFillProcessingMode();
|
|
1795
|
-
const processValidFills = async (fillsToSync) => {
|
|
1796
|
-
let resolvedOrders = [];
|
|
1797
|
-
if (fillMode === 'history') {
|
|
1798
|
-
this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (history mode)`, 'info');
|
|
1799
|
-
// Batch mode for 2+ fills: acquires _gridLock once, batches drift refetch
|
|
1800
|
-
if (fillsToSync.length >= 2) {
|
|
1801
|
-
const batchResult = await this.manager.syncFromFillHistoryBatch(fillsToSync, {
|
|
1802
|
-
persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
|
|
1803
|
-
});
|
|
1804
|
-
for (const fill of fillsToSync) {
|
|
1805
|
-
const fillKey = buildFillKey({
|
|
1806
|
-
orderId: fill?.op?.[1]?.order_id,
|
|
1807
|
-
blockNum: fill?.block_num,
|
|
1808
|
-
historyId: fill?.id
|
|
1809
|
-
});
|
|
1810
|
-
if (fillKey)
|
|
1811
|
-
pendingFillKeysForCurrentCycle.add(fillKey);
|
|
1812
|
-
}
|
|
1813
|
-
if (batchResult.filledOrders)
|
|
1814
|
-
resolvedOrders.push(...batchResult.filledOrders);
|
|
1815
|
-
if (batchResult.requiresOpenOrdersSync)
|
|
1816
|
-
requiresOpenOrdersSync = true;
|
|
1817
|
-
if (batchResult.ghostOrderIds?.length > 0) {
|
|
1818
|
-
for (const id of batchResult.ghostOrderIds) {
|
|
1819
|
-
pendingGhostOrders.add(id);
|
|
1820
|
-
}
|
|
1821
|
-
}
|
|
1822
|
-
}
|
|
1823
|
-
else {
|
|
1824
|
-
// Single fill: use individual per-fill path
|
|
1825
|
-
for (const fill of fillsToSync) {
|
|
1826
|
-
const resultHistory = await this.manager.syncFromFillHistory(fill, {
|
|
1827
|
-
persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
|
|
1828
|
-
});
|
|
1829
|
-
const fillKey = buildFillKey({
|
|
1830
|
-
orderId: fill?.op?.[1]?.order_id,
|
|
1831
|
-
blockNum: fill?.block_num,
|
|
1832
|
-
historyId: fill?.id
|
|
1833
|
-
});
|
|
1834
|
-
if (fillKey)
|
|
1835
|
-
pendingFillKeysForCurrentCycle.add(fillKey);
|
|
1836
|
-
// Dust is handled by post-fill detection below.
|
|
1837
|
-
if (resultHistory.filledOrders)
|
|
1838
|
-
resolvedOrders.push(...resultHistory.filledOrders);
|
|
1839
|
-
if (resultHistory.requiresOpenOrdersSync)
|
|
1840
|
-
requiresOpenOrdersSync = true;
|
|
1841
|
-
if (resultHistory.ghostOrderId)
|
|
1842
|
-
pendingGhostOrders.add(resultHistory.ghostOrderId);
|
|
1843
|
-
}
|
|
1844
|
-
}
|
|
1845
|
-
}
|
|
1846
|
-
if (fillMode !== 'history' || requiresOpenOrdersSync) {
|
|
1847
|
-
if (fillMode === 'history' && requiresOpenOrdersSync) {
|
|
1848
|
-
this.manager.logger.log('Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
|
|
1849
|
-
}
|
|
1850
|
-
this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (open orders mode)`, 'info');
|
|
1851
|
-
const chainOpenOrders = await chainOrders.readOpenOrders(this.account);
|
|
1852
|
-
const resultOpenOrders = await this.manager.syncFromOpenOrders(chainOpenOrders);
|
|
1853
|
-
// Dust is handled by post-fill detection below.
|
|
1854
|
-
if (resultOpenOrders.filledOrders)
|
|
1855
|
-
resolvedOrders.push(...resultOpenOrders.filledOrders);
|
|
1856
|
-
if (resultOpenOrders.ordersNeedingCorrection)
|
|
1857
|
-
ordersNeedingCorrection.push(...resultOpenOrders.ordersNeedingCorrection);
|
|
1858
|
-
}
|
|
1859
|
-
return resolvedOrders;
|
|
1860
|
-
};
|
|
1861
|
-
this.manager.pauseFundRecalc();
|
|
1862
|
-
try {
|
|
1863
|
-
// FIX 1: Block-level fill batching — group valid fills by block
|
|
1864
|
-
// and process each block group as a unit. This prevents slot
|
|
1865
|
-
// collisions when fills from the same block (original + replacement
|
|
1866
|
-
// orders both filled) arrive in different sync batches. Processing
|
|
1867
|
-
// all fills from a block together lets the sync engine see every
|
|
1868
|
-
// fill for overlapping slots simultaneously.
|
|
1869
|
-
const fillsByBlock = new Map();
|
|
1870
|
-
const fillsWithoutBlock = [];
|
|
1871
|
-
for (const fill of validFills) {
|
|
1872
|
-
if (fill.block_num != null) {
|
|
1873
|
-
const list = fillsByBlock.get(fill.block_num);
|
|
1874
|
-
if (list)
|
|
1875
|
-
list.push(fill);
|
|
1876
|
-
else
|
|
1877
|
-
fillsByBlock.set(fill.block_num, [fill]);
|
|
1878
|
-
}
|
|
1879
|
-
else {
|
|
1880
|
-
fillsWithoutBlock.push(fill);
|
|
1881
|
-
}
|
|
1882
|
-
}
|
|
1883
|
-
// Process block groups in ascending block order so the sync
|
|
1884
|
-
// engine sees a deterministic, chronological fill sequence.
|
|
1885
|
-
const sortedBlocks = [...fillsByBlock.keys()].sort((a, b) => a - b);
|
|
1886
|
-
const accumulatedOrders = [];
|
|
1887
|
-
let anyRequiresSync = false;
|
|
1888
|
-
// Capture whether any fill was skipped during validation
|
|
1889
|
-
// (e.g. missing history ID) and set the sync flag. The
|
|
1890
|
-
// per-block reset below would lose this state since those
|
|
1891
|
-
// fills were filtered out of validFills and never enter
|
|
1892
|
-
// any block group.
|
|
1893
|
-
const initialRequiresSync = requiresOpenOrdersSync;
|
|
1894
|
-
for (const blockNum of sortedBlocks) {
|
|
1895
|
-
// Reset requiresOpenOrdersSync per block group so one
|
|
1896
|
-
// block's history-id gap doesn't force the next block
|
|
1897
|
-
// into an unnecessary open-orders snapshot re-fetch.
|
|
1898
|
-
requiresOpenOrdersSync = false;
|
|
1899
|
-
this.manager.logger.log(`[FILL-BLOCK] Processing ${fillsByBlock.get(blockNum).length} fill(s) from block ${blockNum}`, 'debug');
|
|
1900
|
-
const blockResult = await processValidFills(fillsByBlock.get(blockNum));
|
|
1901
|
-
accumulatedOrders.push(...blockResult);
|
|
1902
|
-
if (requiresOpenOrdersSync)
|
|
1903
|
-
anyRequiresSync = true;
|
|
1904
|
-
}
|
|
1905
|
-
// Preserve both per-block sync flags and the initial flag
|
|
1906
|
-
// from filtered-out fills (missing history ID).
|
|
1907
|
-
requiresOpenOrdersSync = anyRequiresSync || initialRequiresSync;
|
|
1908
|
-
if (fillsWithoutBlock.length > 0) {
|
|
1909
|
-
this.manager.logger.log(`[FILL-BLOCK] Processing ${fillsWithoutBlock.length} fill(s) without block info`, 'debug');
|
|
1910
|
-
const noBlockResult = await processValidFills(fillsWithoutBlock);
|
|
1911
|
-
accumulatedOrders.push(...noBlockResult);
|
|
1912
|
-
if (requiresOpenOrdersSync)
|
|
1913
|
-
anyRequiresSync = true;
|
|
1914
|
-
}
|
|
1915
|
-
// If a fill was filtered out during validation (e.g. missing
|
|
1916
|
-
// history ID) and no block group triggered the open-orders
|
|
1917
|
-
// sync, run it now so the grid state is reconciled.
|
|
1918
|
-
if (requiresOpenOrdersSync && !anyRequiresSync) {
|
|
1919
|
-
this.manager.logger.log('[FILL-BLOCK] Running open-orders sync for fills with missing history identifiers', 'warn');
|
|
1920
|
-
const fallbackOrders = await processValidFills([]);
|
|
1921
|
-
accumulatedOrders.push(...fallbackOrders);
|
|
1922
|
-
// Flag consumed — variable goes out of scope on
|
|
1923
|
-
// next while iteration.
|
|
1924
|
-
}
|
|
1925
|
-
allFilledOrders = accumulatedOrders;
|
|
1926
|
-
// 4. Handle Price Corrections
|
|
1927
|
-
if (ordersNeedingCorrection.length > 0) {
|
|
1928
|
-
const correctionResult = await correctAllPriceMismatches(this.manager, this.account, this.privateKey, chainOrders);
|
|
1929
|
-
if (correctionResult.failed > 0)
|
|
1930
|
-
this.manager.logger.log(`${correctionResult.failed} corrections failed`, 'error');
|
|
1931
|
-
}
|
|
1932
|
-
}
|
|
1933
|
-
finally {
|
|
1934
|
-
// 4b. Cancel orphaned chain orders detected by other-side rounding.
|
|
1935
|
-
// These orders have a tiny residual that the bot treated as fully
|
|
1936
|
-
// filled, but the blockchain did not close the order. We send a
|
|
1937
|
-
// cancel tx to clean up the zombie order on chain.
|
|
1938
|
-
// Runs in the finally so it still fires even if processValidFills
|
|
1939
|
-
// throws after collecting ghost IDs into pendingGhostOrders.
|
|
1940
|
-
//
|
|
1941
|
-
// OPTIMIZATION: Batch all new ghost cancels into one executeBatch
|
|
1942
|
-
// call with multiple cancelOrder ops, reducing per-order tx fees
|
|
1943
|
-
// and broadcast overhead. Falls back to individual cancels on
|
|
1944
|
-
// batch failure.
|
|
1945
|
-
if (pendingGhostOrders.size > 0) {
|
|
1946
|
-
if (!this._ghostOrderCancelAttempted)
|
|
1947
|
-
this._ghostOrderCancelAttempted = new Set();
|
|
1948
|
-
const newGhostIds = [...pendingGhostOrders].filter(id => !this._ghostOrderCancelAttempted.has(id));
|
|
1949
|
-
if (newGhostIds.length > 0) {
|
|
1950
|
-
const MAX_OPS_PER_TX = 200;
|
|
1951
|
-
let batchFailed = false;
|
|
1952
|
-
// Build cancel ops with per-ID error tolerance.
|
|
1953
|
-
// Index correlates 1:1 with newGhostIds (allSettled preserves order).
|
|
1954
|
-
const buildResults = await Promise.allSettled(newGhostIds.map(id => chainOrders.buildCancelOrderOp(this.account, id)));
|
|
1955
|
-
const cancelOps = [];
|
|
1956
|
-
for (let i = 0; i < buildResults.length; i++) {
|
|
1957
|
-
const result = buildResults[i];
|
|
1958
|
-
const id = newGhostIds[i];
|
|
1959
|
-
if (result.status === 'fulfilled') {
|
|
1960
|
-
cancelOps.push(result.value);
|
|
1961
|
-
}
|
|
1962
|
-
else {
|
|
1963
|
-
this.manager.logger.log(`[SYNC] Failed to build cancel op for ghost order ${id}: ${result.reason?.message || result.reason}`, 'warn');
|
|
1964
|
-
}
|
|
1965
|
-
}
|
|
1966
|
-
// Chunk into batches respecting MAX_OPS_PER_TX
|
|
1967
|
-
for (let i = 0; i < cancelOps.length; i += MAX_OPS_PER_TX) {
|
|
1968
|
-
const chunk = cancelOps.slice(i, i + MAX_OPS_PER_TX);
|
|
1969
|
-
const batchIds = newGhostIds.slice(i, i + MAX_OPS_PER_TX);
|
|
1970
|
-
try {
|
|
1971
|
-
this.manager.logger.log(`[SYNC] Batch-cancelling ${chunk.length} orphaned chain order(s) ` +
|
|
1972
|
-
`(batch ${Math.floor(i / MAX_OPS_PER_TX) + 1}/${Math.ceil(cancelOps.length / MAX_OPS_PER_TX)})`, 'info');
|
|
1973
|
-
await chainOrders.executeBatch(this.account, this.privateKey, chunk);
|
|
1974
|
-
// Mark successfully cancelled IDs immediately
|
|
1975
|
-
for (const id of batchIds) {
|
|
1976
|
-
this._ghostOrderCancelAttempted.add(id);
|
|
1977
|
-
}
|
|
1978
|
-
}
|
|
1979
|
-
catch (batchErr) {
|
|
1980
|
-
batchFailed = true;
|
|
1981
|
-
this.manager.logger.log(`[SYNC] Batch ghost cancel failed for chunk ${Math.floor(i / MAX_OPS_PER_TX) + 1} ` +
|
|
1982
|
-
`(${chunk.length} orders): ${batchErr?.message || batchErr}`, 'warn');
|
|
1983
|
-
}
|
|
1984
|
-
}
|
|
1985
|
-
// Log overall success if all chunks and all builds succeeded
|
|
1986
|
-
if (!batchFailed && cancelOps.length === newGhostIds.length) {
|
|
1987
|
-
this.manager.logger.log(`[SYNC] Successfully batch-cancelled ${newGhostIds.length} orphaned chain order(s)`, 'info');
|
|
1988
|
-
}
|
|
1989
|
-
// Fallback: individual cancels for IDs that were not marked successful
|
|
1990
|
-
for (const ghostOrderId of newGhostIds) {
|
|
1991
|
-
if (this._ghostOrderCancelAttempted.has(ghostOrderId))
|
|
1992
|
-
continue;
|
|
1993
|
-
try {
|
|
1994
|
-
this.manager.logger.log(`[SYNC] Cancelling orphaned chain order ${ghostOrderId} (other-side full-fill residual)`, 'info');
|
|
1995
|
-
await chainOrders.cancelOrder(this.account, this.privateKey, ghostOrderId);
|
|
1996
|
-
this._ghostOrderCancelAttempted.add(ghostOrderId);
|
|
1997
|
-
}
|
|
1998
|
-
catch (err) {
|
|
1999
|
-
this.manager.logger.log(`[SYNC] Failed to cancel orphaned order ${ghostOrderId}: ${err?.message || err}`, 'warn');
|
|
2000
|
-
}
|
|
2001
|
-
}
|
|
2002
|
-
}
|
|
2003
|
-
}
|
|
2004
|
-
await this.manager.resumeFundRecalc();
|
|
2005
|
-
}
|
|
2006
|
-
// Refresh dynamic weight distribution before processing fills
|
|
2007
|
-
// so the rebalance uses the latest market adapter weights, not
|
|
2008
|
-
// stale values from the last periodic refresh cycle.
|
|
2009
|
-
// Lightweight: reads local JSON snapshot file.
|
|
2010
|
-
this._refreshDynamicWeightDistribution('fill queue');
|
|
2011
|
-
// 5. Fixed-Cap Fill Rebalance
|
|
2012
|
-
// - 1..MAX_FILL_BATCH_SIZE fills: unified full-set planning
|
|
2013
|
-
// - larger bursts: fixed-size chunking at MAX_FILL_BATCH_SIZE
|
|
2014
|
-
if (allFilledOrders.length > 0) {
|
|
2015
|
-
const result = await this._processFillsWithBatching(allFilledOrders, null, 'fill set');
|
|
2016
|
-
let abortedFillCycle = result.aborted;
|
|
2017
|
-
if (!abortedFillCycle) {
|
|
2018
|
-
const batchFillKeys = new Set(allFilledOrders.map(filledOrder => buildFillKey({
|
|
2019
|
-
orderId: filledOrder?.orderId,
|
|
2020
|
-
blockNum: filledOrder?.blockNum,
|
|
2021
|
-
historyId: filledOrder?.historyId
|
|
2022
|
-
})).filter(Boolean));
|
|
2023
|
-
await this._flushProcessedFillPersistenceForKeys(batchFillKeys, 'fill-batch-committed');
|
|
2024
|
-
}
|
|
2025
|
-
else {
|
|
2026
|
-
this.manager.logger.log('[FILL-DEDUP] Fill cycle aborted; fill key persistence guarded under abort path.', 'warn');
|
|
2027
|
-
}
|
|
2028
|
-
// 6. Rebalance Recovery Loop (Sequential Extensions)
|
|
2029
|
-
// DISABLED FOR SEQUENTIAL: Each sequential fill already triggers a full rebalance with proper
|
|
2030
|
-
// boundary shift. An additional recovery loop with EMPTY fills causes the boundary to remain
|
|
2031
|
-
// at the last fill's position, leading to wrong operation types (updates instead of rotations)
|
|
2032
|
-
// and operations on the wrong side.
|
|
2033
|
-
//
|
|
2034
|
-
// In the future, recovery loop can be re-enabled for single fills if needed, but ONLY
|
|
2035
|
-
// if it passes the actual fills to processFilledOrders so the boundary shifts correctly.
|
|
2036
|
-
// For now: Each fill = full rebalance with boundary shift = complete correction in one pass.
|
|
2037
|
-
// CRITICAL: Do NOT run spread correction here during sequential fill processing.
|
|
2038
|
-
// The rebalance from each fill should maintain spread naturally. Running spread correction
|
|
2039
|
-
// immediately after creates new orders that may get filled by market before next cycle,
|
|
2040
|
-
// causing cascading fills and potentially SPREAD slots becoming PARTIAL (error condition).
|
|
2041
|
-
// Spread correction runs in the main loop instead.
|
|
2042
|
-
const fullFillCount = allFilledOrders.filter(o => o && o.isPartial !== true).length;
|
|
2043
|
-
const hasAnyFills = allFilledOrders.some(o => o);
|
|
2044
|
-
const shouldRunPostFillChecks = !abortedFillCycle && fullFillCount > 0;
|
|
2045
|
-
const shouldRunDustDetection = !abortedFillCycle && hasAnyFills;
|
|
2046
|
-
if (shouldRunDustDetection) {
|
|
2047
|
-
const healthResult = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
|
|
2048
|
-
const allDust = [
|
|
2049
|
-
...(healthResult.buyDustOrders || []),
|
|
2050
|
-
...(healthResult.sellDustOrders || []),
|
|
2051
|
-
];
|
|
2052
|
-
if (allDust.length > 0) {
|
|
2053
|
-
const dustCancelResult = await this._cancelDustOrders({
|
|
2054
|
-
buy: healthResult.buyDustOrders,
|
|
2055
|
-
sell: healthResult.sellDustOrders,
|
|
2056
|
-
});
|
|
2057
|
-
if (dustCancelResult?.batchResult?.aborted) {
|
|
2058
|
-
abortedFillCycle = true;
|
|
2059
|
-
}
|
|
2060
|
-
}
|
|
2061
|
-
}
|
|
2062
|
-
// Run grid maintenance after fills to rebuild degraded grid.
|
|
2063
|
-
// CRITICAL FIX (commit a946c33): Replaced inline divergence checks with centralized
|
|
2064
|
-
// _runGridMaintenance call to ensure pipeline protection applies consistently.
|
|
2065
|
-
// Before: Divergence checks ran immediately after fills, causing race-to-resize
|
|
2066
|
-
// After: Grid maintenance waits for isPipelineEmpty() before structural changes
|
|
2067
|
-
// Run only when the cycle contains at least one full fill.
|
|
2068
|
-
if (shouldRunPostFillChecks && !abortedFillCycle) {
|
|
2069
|
-
await this._runGridMaintenance('post-fill');
|
|
2070
|
-
}
|
|
2071
|
-
}
|
|
2072
|
-
else if (pendingFillKeysForCurrentCycle.size > 0) {
|
|
2073
|
-
await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, 'fill-batch-no-rotations');
|
|
2074
|
-
}
|
|
2075
|
-
this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
|
|
2076
|
-
await retryPersistenceIfNeeded(this.manager);
|
|
2077
|
-
// Periodically clean up old fill records after processing N fills.
|
|
2078
|
-
// Counter is protected by _fillProcessingLock during fill consumption.
|
|
2079
|
-
this._fillCleanupCounter += validFills.length;
|
|
2080
|
-
const cleanupThreshold = MAINTENANCE.CLEANUP_PROBABILITY > 0 && MAINTENANCE.CLEANUP_PROBABILITY < 1
|
|
2081
|
-
? Math.floor(1 / MAINTENANCE.CLEANUP_PROBABILITY)
|
|
2082
|
-
: 100; // Default: every 100 fills
|
|
2083
|
-
if (this._fillCleanupCounter >= cleanupThreshold) {
|
|
2084
|
-
try {
|
|
2085
|
-
await this.accountOrders.cleanOldProcessedFills(TIMING.FILL_RECORD_RETENTION_MS);
|
|
2086
|
-
this._fillCleanupCounter = 0; // Reset counter after cleanup (success or retry on next batch if failed)
|
|
2087
|
-
}
|
|
2088
|
-
catch (err) {
|
|
2089
|
-
this.manager?.logger?.log(`Warning: Fill cleanup failed (will retry): ${err.message}`, 'warn');
|
|
2090
|
-
}
|
|
2091
|
-
}
|
|
2092
|
-
// Update metrics
|
|
2093
|
-
this._metrics.fillsProcessed += validFills.length;
|
|
2094
|
-
this._metrics.fillProcessingTimeMs += Date.now() - batchStartTime;
|
|
2095
|
-
// Prune expired stale-cleaned order IDs after each processing cycle.
|
|
2096
|
-
if (this._staleCleanedOrderIds.size > 0) {
|
|
2097
|
-
const now = Date.now();
|
|
2098
|
-
let prunedCount = 0;
|
|
2099
|
-
for (const [orderId, markedAt] of this._staleCleanedOrderIds) {
|
|
2100
|
-
if (now - markedAt > this._staleCleanupRetentionMs) {
|
|
2101
|
-
this._staleCleanedOrderIds.delete(orderId);
|
|
2102
|
-
prunedCount++;
|
|
2103
|
-
}
|
|
2104
|
-
}
|
|
2105
|
-
if (prunedCount > 0) {
|
|
2106
|
-
this.manager.logger.log(`[STALE-CLEANUP] Pruned ${prunedCount} expired stale-cleaned order IDs ` +
|
|
2107
|
-
`(retention=${this._staleCleanupRetentionMs}ms, remaining=${this._staleCleanedOrderIds.size})`, 'debug');
|
|
2108
|
-
}
|
|
2109
|
-
}
|
|
2110
|
-
} // End while(_incomingFillQueue)
|
|
2111
|
-
this._markGridActivity('fill processing end');
|
|
2112
|
-
// Reset the fill-consumer watchdog on the success path. The
|
|
2113
|
-
// counter is only incremented by _scheduleFillConsumerRestart's
|
|
2114
|
-
// catch handler; without this reset, a fill pattern of
|
|
2115
|
-
// F-S-F-S-F-S-F-S-F would still reach the max and the consumer
|
|
2116
|
-
// would stop being re-scheduled.
|
|
2117
|
-
this._consecutiveConsumeFailures = 0;
|
|
2118
|
-
this._consumeFailureFirstAt = 0;
|
|
2119
|
-
});
|
|
2120
|
-
}
|
|
2121
|
-
catch (err) {
|
|
2122
|
-
const isCredentialOutage = this._isCredentialDaemonError(err);
|
|
2123
|
-
if (pendingFillKeysForCurrentCycle.size > 0) {
|
|
2124
|
-
const flushReason = isCredentialOutage
|
|
2125
|
-
? 'credential-outage-verified-fills'
|
|
2126
|
-
: 'fill-cycle-error-verified-fills';
|
|
2127
|
-
if (isCredentialOutage) {
|
|
2128
|
-
this._credentialRecoveryNeeded = true;
|
|
2129
|
-
this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
|
|
2130
|
-
}
|
|
2131
|
-
try {
|
|
2132
|
-
await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, flushReason, { throwOnError: true });
|
|
2133
|
-
const credentialSuffix = isCredentialOutage
|
|
2134
|
-
? '; grid persistence is suspended until recovery'
|
|
2135
|
-
: '';
|
|
2136
|
-
this.manager?.logger?.log?.(`[FILL-DEDUP] Persisted ${pendingFillKeysForCurrentCycle.size} verified processed-fill write(s) after fill cycle error${credentialSuffix}.`, isCredentialOutage ? 'warn' : 'info');
|
|
2137
|
-
}
|
|
2138
|
-
catch (flushErr) {
|
|
2139
|
-
this.manager?.logger?.log?.(`[FILL-DEDUP] Failed to persist verified fill keys during fill error handling: ${flushErr.message}`, 'warn');
|
|
2140
|
-
}
|
|
2141
|
-
}
|
|
2142
|
-
if (isCredentialOutage && pendingFillKeysForCurrentCycle.size === 0) {
|
|
2143
|
-
this._credentialRecoveryNeeded = true;
|
|
2144
|
-
this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
|
|
2145
|
-
}
|
|
2146
|
-
this._log(`Error processing fills: ${err.message}`, 'error');
|
|
2147
|
-
if (err.stack)
|
|
2148
|
-
this._log(err.stack, 'error');
|
|
2149
|
-
}
|
|
2150
|
-
// Post-processing: If new fills arrived while processing, schedule another cycle
|
|
2151
|
-
// SAFE: Done outside lock context, no async work in finally block
|
|
2152
|
-
if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
|
|
2153
|
-
this._scheduleFillConsumerRestart(chainOrders);
|
|
2154
|
-
}
|
|
648
|
+
return DexbotFillRuntime.consumeFillQueue(this, chainOrders);
|
|
2155
649
|
}
|
|
2156
650
|
/**
|
|
2157
651
|
* Process fills during bootstrap phase using the standard fill pipeline.
|
|
@@ -2171,98 +665,7 @@ class DEXBot {
|
|
|
2171
665
|
* @returns {Promise<void>}
|
|
2172
666
|
*/
|
|
2173
667
|
async _processFillsWithBootstrapMode(chainOrders) {
|
|
2174
|
-
|
|
2175
|
-
return;
|
|
2176
|
-
const startTime = Date.now();
|
|
2177
|
-
const fills = this._incomingFillQueue.splice(0);
|
|
2178
|
-
const validFills = [];
|
|
2179
|
-
const processedFillKeys = new Set();
|
|
2180
|
-
let requiresOpenOrdersSync = false;
|
|
2181
|
-
// 1. Validate and deduplicate fills
|
|
2182
|
-
for (const fill of fills) {
|
|
2183
|
-
if (!fill || fill.op?.[0] !== 4)
|
|
2184
|
-
continue;
|
|
2185
|
-
const fillOp = fill.op[1];
|
|
2186
|
-
const gridOrder = this.manager.orders.get(fillOp.order_id) ||
|
|
2187
|
-
Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
|
|
2188
|
-
if (!gridOrder) {
|
|
2189
|
-
// CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
|
|
2190
|
-
// This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
|
|
2191
|
-
let orphanFillKey = buildFillKey(fill);
|
|
2192
|
-
if (!orphanFillKey) {
|
|
2193
|
-
orphanFillKey = this._buildOrphanFillFallbackKey(fill);
|
|
2194
|
-
}
|
|
2195
|
-
if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
|
|
2196
|
-
continue;
|
|
2197
|
-
}
|
|
2198
|
-
this.manager.logger.log(`[BOOTSTRAP] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
|
|
2199
|
-
const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
|
|
2200
|
-
context: 'BOOTSTRAP'
|
|
2201
|
-
});
|
|
2202
|
-
if (accountingResult.status === 'missing_key') {
|
|
2203
|
-
requiresOpenOrdersSync = true;
|
|
2204
|
-
}
|
|
2205
|
-
continue;
|
|
2206
|
-
}
|
|
2207
|
-
const trackedFillKey = buildFillKey(fill);
|
|
2208
|
-
if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
|
|
2209
|
-
continue;
|
|
2210
|
-
}
|
|
2211
|
-
this.manager.lockOrders([gridOrder.id]);
|
|
2212
|
-
try {
|
|
2213
|
-
const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
|
|
2214
|
-
context: 'BOOTSTRAP',
|
|
2215
|
-
replayMessage: (op) => `[BOOTSTRAP] Replay detected for ${op.order_id}; skipping duplicate bootstrap rebalance`
|
|
2216
|
-
});
|
|
2217
|
-
if (accountingResult.status === 'missing_key') {
|
|
2218
|
-
requiresOpenOrdersSync = true;
|
|
2219
|
-
continue;
|
|
2220
|
-
}
|
|
2221
|
-
if (accountingResult.status !== 'applied') {
|
|
2222
|
-
continue;
|
|
2223
|
-
}
|
|
2224
|
-
validFills.push({ ...fill, gridOrder });
|
|
2225
|
-
const fillType = gridOrder.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL';
|
|
2226
|
-
this._log(`[BOOTSTRAP] Fill detected: ${fillType} order (${fillOp.is_maker !== false ? 'maker' : 'taker'})`);
|
|
2227
|
-
}
|
|
2228
|
-
finally {
|
|
2229
|
-
this.manager.unlockOrders([gridOrder.id]);
|
|
2230
|
-
}
|
|
2231
|
-
}
|
|
2232
|
-
if (requiresOpenOrdersSync) {
|
|
2233
|
-
this._log('[BOOTSTRAP] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
|
|
2234
|
-
const bootstrapChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
|
|
2235
|
-
const syncResult = await this.manager.syncFromOpenOrders(bootstrapChainOpenOrders);
|
|
2236
|
-
if (syncResult.filledOrders?.length > 0) {
|
|
2237
|
-
const queuedOrderIds = new Set(validFills.map(fill => fill?.gridOrder?.orderId).filter(Boolean));
|
|
2238
|
-
for (const filledOrder of syncResult.filledOrders) {
|
|
2239
|
-
if (!filledOrder?.orderId || queuedOrderIds.has(filledOrder.orderId))
|
|
2240
|
-
continue;
|
|
2241
|
-
validFills.push({ gridOrder: filledOrder });
|
|
2242
|
-
queuedOrderIds.add(filledOrder.orderId);
|
|
2243
|
-
}
|
|
2244
|
-
}
|
|
2245
|
-
}
|
|
2246
|
-
await this._flushProcessedFillPersistence('bootstrap-batch');
|
|
2247
|
-
if (validFills.length === 0)
|
|
2248
|
-
return;
|
|
2249
|
-
// 2. Process fills through the standard fill pipeline.
|
|
2250
|
-
// This handles same-side replacement, symmetric rebalance, and dust cleanup
|
|
2251
|
-
// via the same path used by the post-reset flow.
|
|
2252
|
-
try {
|
|
2253
|
-
this._log(`[BOOTSTRAP] Processing ${validFills.length} fill(s) through standard pipeline`, 'info');
|
|
2254
|
-
const filledOrders = validFills.map(f => f.gridOrder);
|
|
2255
|
-
const result = await this._processFillsWithBatching(filledOrders, new Set(), '[BOOTSTRAP] fill processing');
|
|
2256
|
-
if (result.aborted) {
|
|
2257
|
-
this._warn('[BOOTSTRAP] Aborted batch due to illegal state; skipping grid persistence this cycle');
|
|
2258
|
-
}
|
|
2259
|
-
this._metrics.fillsProcessed += validFills.length;
|
|
2260
|
-
this._metrics.fillProcessingTimeMs += Date.now() - startTime;
|
|
2261
|
-
}
|
|
2262
|
-
catch (err) {
|
|
2263
|
-
this._warn(`[BOOTSTRAP] Error processing fills: ${err.message}`);
|
|
2264
|
-
this.manager.logger.log(`[BOOTSTRAP] Fill error: ${err.message}`, 'error');
|
|
2265
|
-
}
|
|
668
|
+
return DexbotFillRuntime.processFillsWithBootstrapMode(this, chainOrders);
|
|
2266
669
|
}
|
|
2267
670
|
/**
|
|
2268
671
|
* Set up account identifier and configure global context.
|
|
@@ -2332,43 +735,7 @@ class DEXBot {
|
|
|
2332
735
|
* @returns {Promise<void>}
|
|
2333
736
|
*/
|
|
2334
737
|
async placeInitialOrders() {
|
|
2335
|
-
|
|
2336
|
-
const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
|
|
2337
|
-
this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
|
|
2338
|
-
this.manager.accountOrders = this.accountOrders;
|
|
2339
|
-
}
|
|
2340
|
-
this._wireStructuralGridResyncRequest();
|
|
2341
|
-
this.manager.startBootstrap();
|
|
2342
|
-
try {
|
|
2343
|
-
try {
|
|
2344
|
-
const botFunds = this.config && this.config.botFunds ? this.config.botFunds : {};
|
|
2345
|
-
const needsPercent = (v) => typeof v === 'string' && v.includes('%');
|
|
2346
|
-
if ((needsPercent(botFunds.buy) || needsPercent(botFunds.sell)) && (this.accountId || this.account)) {
|
|
2347
|
-
if (typeof this.manager._fetchAccountBalancesAndSetTotals === 'function') {
|
|
2348
|
-
await this.manager._fetchAccountBalancesAndSetTotals();
|
|
2349
|
-
}
|
|
2350
|
-
}
|
|
2351
|
-
}
|
|
2352
|
-
catch (errFetch) {
|
|
2353
|
-
this._warn(`Could not fetch account totals before initializing grid: ${errFetch && errFetch.message ? errFetch.message : errFetch}`);
|
|
2354
|
-
}
|
|
2355
|
-
await Grid.initializeGrid(this.manager);
|
|
2356
|
-
if (this.config.dryRun) {
|
|
2357
|
-
this.manager.logger.log('Dry run enabled, skipping on-chain order placement.', 'info');
|
|
2358
|
-
await this.manager.persistGrid();
|
|
2359
|
-
return;
|
|
2360
|
-
}
|
|
2361
|
-
this.manager.logger.log('Placing initial orders on-chain...', 'info');
|
|
2362
|
-
const ordersToActivate = this.manager.getInitialOrdersToActivate();
|
|
2363
|
-
const orderGroups = this._buildOutsideInPairGroupsForOrders(ordersToActivate);
|
|
2364
|
-
for (const group of orderGroups) {
|
|
2365
|
-
await this.updateOrdersOnChainPlan({ ordersToPlace: group });
|
|
2366
|
-
}
|
|
2367
|
-
await this.manager.persistGrid();
|
|
2368
|
-
}
|
|
2369
|
-
finally {
|
|
2370
|
-
this.manager.finishBootstrap();
|
|
2371
|
-
}
|
|
738
|
+
return DexbotStartupRuntime.placeInitialOrdersImpl(this);
|
|
2372
739
|
}
|
|
2373
740
|
/**
|
|
2374
741
|
* Build outside-in pair groups for initial order placement.
|
|
@@ -2376,11 +743,7 @@ class DEXBot {
|
|
|
2376
743
|
* @returns {Array<Array<Object>>} Grouped order arrays
|
|
2377
744
|
*/
|
|
2378
745
|
_buildOutsideInPairGroupsForOrders(orders) {
|
|
2379
|
-
return
|
|
2380
|
-
isValid: Boolean,
|
|
2381
|
-
getType: o => o.type,
|
|
2382
|
-
getPrice: o => o.price,
|
|
2383
|
-
});
|
|
746
|
+
return cowRuntime.buildOutsideInPairGroupsForOrders(orders);
|
|
2384
747
|
}
|
|
2385
748
|
/**
|
|
2386
749
|
* Build outside-in pair groups for create entry contexts.
|
|
@@ -2388,11 +751,7 @@ class DEXBot {
|
|
|
2388
751
|
* @returns {Array<Array<Object>>} Grouped entry arrays
|
|
2389
752
|
*/
|
|
2390
753
|
_buildOutsideInPairGroupsForCreateEntries(createEntries) {
|
|
2391
|
-
return
|
|
2392
|
-
isValid: e => Boolean(e?.context?.order),
|
|
2393
|
-
getType: e => e.context.order.type,
|
|
2394
|
-
getPrice: e => e.context.order.price,
|
|
2395
|
-
});
|
|
754
|
+
return cowRuntime.buildOutsideInPairGroupsForCreateEntries(createEntries);
|
|
2396
755
|
}
|
|
2397
756
|
/**
|
|
2398
757
|
* Resolve the centralized fill batch cap.
|
|
@@ -2408,19 +767,7 @@ class DEXBot {
|
|
|
2408
767
|
* @returns {Array} Array of operation result entries
|
|
2409
768
|
*/
|
|
2410
769
|
_extractOperationResults(result, warnContext = '') {
|
|
2411
|
-
|
|
2412
|
-
if (Array.isArray(extracted))
|
|
2413
|
-
return extracted;
|
|
2414
|
-
if (result) {
|
|
2415
|
-
const resultType = Array.isArray(result) ? 'array' : typeof result;
|
|
2416
|
-
const keySummary = (resultType === 'object' && !Array.isArray(result))
|
|
2417
|
-
? Object.keys(result).slice(0, 8).join(',')
|
|
2418
|
-
: '';
|
|
2419
|
-
const contextSuffix = warnContext ? ` (${warnContext})` : '';
|
|
2420
|
-
const keysSuffix = keySummary ? `; keys=[${keySummary}]` : '';
|
|
2421
|
-
this.manager?.logger?.log(`[COW] Unrecognized operation_results shape${contextSuffix}; defaulting to empty results. resultType=${resultType}${keysSuffix}`, 'warn');
|
|
2422
|
-
}
|
|
2423
|
-
return [];
|
|
770
|
+
return cowRuntime.extractOperationResults(result, warnContext, this.manager?.logger?.log?.bind(this.manager?.logger));
|
|
2424
771
|
}
|
|
2425
772
|
/**
|
|
2426
773
|
* Find CREATE operation contexts whose broadcast result did not include a chain order id.
|
|
@@ -2430,19 +777,7 @@ class DEXBot {
|
|
|
2430
777
|
* @returns {Array<{index:number, ctx:Object}>} Missing create result contexts.
|
|
2431
778
|
*/
|
|
2432
779
|
_findMissingCreateResultContexts(operationResults, opContexts) {
|
|
2433
|
-
|
|
2434
|
-
if (!Array.isArray(opContexts))
|
|
2435
|
-
return missing;
|
|
2436
|
-
for (let i = 0; i < opContexts.length; i++) {
|
|
2437
|
-
const ctx = opContexts[i];
|
|
2438
|
-
if (ctx?.kind !== 'create')
|
|
2439
|
-
continue;
|
|
2440
|
-
const chainOrderId = operationResults?.[i]?.[1];
|
|
2441
|
-
if (!chainOrderId || !/^1\.7\.\d+$/.test(String(chainOrderId))) {
|
|
2442
|
-
missing.push({ index: i, ctx });
|
|
2443
|
-
}
|
|
2444
|
-
}
|
|
2445
|
-
return missing;
|
|
780
|
+
return cowRuntime.findMissingCreateResultContexts(operationResults, opContexts);
|
|
2446
781
|
}
|
|
2447
782
|
/**
|
|
2448
783
|
* Run an immediate chain sync after a successful CREATE broadcast returned incomplete ids.
|
|
@@ -2456,42 +791,7 @@ class DEXBot {
|
|
|
2456
791
|
* @returns {Promise<void>}
|
|
2457
792
|
*/
|
|
2458
793
|
async _recoverAfterMissingCreateResults(reason = 'missing create operation results') {
|
|
2459
|
-
|
|
2460
|
-
const accountRef = this.accountId || this.account?.id || this.account;
|
|
2461
|
-
if (!accountRef || !this.manager || !chainOrders?.readOpenOrders) {
|
|
2462
|
-
this.manager?.logger?.log?.(`[COW] Recovery sync unavailable after ${reason}`, 'warn');
|
|
2463
|
-
return;
|
|
2464
|
-
}
|
|
2465
|
-
const preRecoveryMissingCreateBlockers = Array.isArray(this.manager._lastUnmatchedChainOrders)
|
|
2466
|
-
? this.manager._lastUnmatchedChainOrders
|
|
2467
|
-
.filter(order => order?.reason === 'missing-create-result')
|
|
2468
|
-
.map(order => ({ ...order }))
|
|
2469
|
-
: [];
|
|
2470
|
-
const openOrders = await chainOrders.readOpenOrders(accountRef);
|
|
2471
|
-
const recoveryResult = await this.manager.syncFromOpenOrders(openOrders, {
|
|
2472
|
-
skipAccounting: false,
|
|
2473
|
-
});
|
|
2474
|
-
this._preserveMissingCreateBlockersAfterRecovery(preRecoveryMissingCreateBlockers, recoveryResult);
|
|
2475
|
-
// Persist any master grid mutations from the recovery sync. The
|
|
2476
|
-
// caller returned from the COW catch handler before reaching the
|
|
2477
|
-
// success-path persistGrid.
|
|
2478
|
-
if (typeof this.manager.persistGrid === 'function') {
|
|
2479
|
-
await this.manager.persistGrid();
|
|
2480
|
-
}
|
|
2481
|
-
}
|
|
2482
|
-
catch (err) {
|
|
2483
|
-
this.manager?.logger?.log?.(`[COW] CRITICAL: Recovery sync failed after ${reason}: ${err.message}`, 'error');
|
|
2484
|
-
if (typeof this.manager?.requestStructuralGridResync === 'function') {
|
|
2485
|
-
try {
|
|
2486
|
-
await this.manager.requestStructuralGridResync(`recovery sync failed after ${reason}`, {
|
|
2487
|
-
error: err.message
|
|
2488
|
-
});
|
|
2489
|
-
}
|
|
2490
|
-
catch (scheduleErr) {
|
|
2491
|
-
this.manager?.logger?.log?.(`[COW] CRITICAL: Failed to schedule structural resync after recovery failure: ${scheduleErr.message}`, 'error');
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
|
-
}
|
|
794
|
+
return cowRuntime.recoverAfterMissingCreateResults(this, reason);
|
|
2495
795
|
}
|
|
2496
796
|
/**
|
|
2497
797
|
* Restore unresolved missing-create blockers after recovery if sync did not adopt them.
|
|
@@ -2501,30 +801,7 @@ class DEXBot {
|
|
|
2501
801
|
* @returns {void}
|
|
2502
802
|
*/
|
|
2503
803
|
_preserveMissingCreateBlockersAfterRecovery(blockers, recoveryResult) {
|
|
2504
|
-
|
|
2505
|
-
return;
|
|
2506
|
-
const adoptedSlotIds = new Set((Array.isArray(recoveryResult?.updatedOrders) ? recoveryResult.updatedOrders : [])
|
|
2507
|
-
.filter(order => order?.id && order?.orderId)
|
|
2508
|
-
.map(order => order.id));
|
|
2509
|
-
const unresolvedBlockers = blockers.filter(blocker => !blocker.slotId || !adoptedSlotIds.has(blocker.slotId));
|
|
2510
|
-
if (unresolvedBlockers.length === 0)
|
|
2511
|
-
return;
|
|
2512
|
-
const currentUnmatched = Array.isArray(this.manager._lastUnmatchedChainOrders)
|
|
2513
|
-
? this.manager._lastUnmatchedChainOrders
|
|
2514
|
-
: [];
|
|
2515
|
-
const currentKeys = new Set(currentUnmatched.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
|
|
2516
|
-
const restored = [...currentUnmatched];
|
|
2517
|
-
for (const blocker of unresolvedBlockers) {
|
|
2518
|
-
const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
|
|
2519
|
-
if (!currentKeys.has(key))
|
|
2520
|
-
restored.push({ ...blocker });
|
|
2521
|
-
}
|
|
2522
|
-
if (restored.length !== currentUnmatched.length) {
|
|
2523
|
-
this.manager._lastUnmatchedChainOrders = restored;
|
|
2524
|
-
this.manager._lastUnmatchedChainOrdersAt = Date.now();
|
|
2525
|
-
this.manager.logger?.log?.(`[COW] Preserving ${restored.length - currentUnmatched.length} missing-create blocker(s) after recovery sync; ` +
|
|
2526
|
-
`chain snapshot did not account for the affected slot(s).`, 'warn');
|
|
2527
|
-
}
|
|
804
|
+
return cowRuntime.preserveMissingCreateBlockersAfterRecovery(this, blockers, recoveryResult);
|
|
2528
805
|
}
|
|
2529
806
|
/**
|
|
2530
807
|
* Merge missing CREATE result contexts into manager._lastUnmatchedChainOrders.
|
|
@@ -2538,42 +815,7 @@ class DEXBot {
|
|
|
2538
815
|
* @returns {void}
|
|
2539
816
|
*/
|
|
2540
817
|
_markMissingCreateResultsAsStructuralBlocker(missingCreateResults) {
|
|
2541
|
-
|
|
2542
|
-
? missingCreateResults.map(item => {
|
|
2543
|
-
const order = item.ctx?.order || {};
|
|
2544
|
-
const fingerprint = [
|
|
2545
|
-
`type=${order.type || 'unknown'}`,
|
|
2546
|
-
`price=${Format.formatPrice6(order.price)}`,
|
|
2547
|
-
`size=${Format.formatAmount(order.size)}`
|
|
2548
|
-
].join(',');
|
|
2549
|
-
return {
|
|
2550
|
-
chainOrderId: 'unknown',
|
|
2551
|
-
type: order.type || null,
|
|
2552
|
-
price: order.price,
|
|
2553
|
-
size: order.size,
|
|
2554
|
-
slotId: order.id || item.ctx?.id || null,
|
|
2555
|
-
reason: 'missing-create-result',
|
|
2556
|
-
operationIndex: item.index,
|
|
2557
|
-
fingerprint,
|
|
2558
|
-
};
|
|
2559
|
-
})
|
|
2560
|
-
: [];
|
|
2561
|
-
if (this.manager && blockers.length > 0) {
|
|
2562
|
-
const existing = Array.isArray(this.manager._lastUnmatchedChainOrders)
|
|
2563
|
-
? this.manager._lastUnmatchedChainOrders
|
|
2564
|
-
: [];
|
|
2565
|
-
const keys = new Set(existing.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
|
|
2566
|
-
const merged = [...existing];
|
|
2567
|
-
for (const blocker of blockers) {
|
|
2568
|
-
const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
|
|
2569
|
-
if (!keys.has(key)) {
|
|
2570
|
-
merged.push(blocker);
|
|
2571
|
-
keys.add(key);
|
|
2572
|
-
}
|
|
2573
|
-
}
|
|
2574
|
-
this.manager._lastUnmatchedChainOrders = merged;
|
|
2575
|
-
this.manager._lastUnmatchedChainOrdersAt = Date.now();
|
|
2576
|
-
}
|
|
818
|
+
return cowRuntime.markMissingCreateResultsAsStructuralBlocker(this, missingCreateResults);
|
|
2577
819
|
}
|
|
2578
820
|
/**
|
|
2579
821
|
* Format an unmatched chain order/blocker for COW logs.
|
|
@@ -2582,7 +824,7 @@ class DEXBot {
|
|
|
2582
824
|
* @returns {string} Compact human-readable diagnostic.
|
|
2583
825
|
*/
|
|
2584
826
|
_formatUnmatchedChainOrderForLog(order) {
|
|
2585
|
-
return
|
|
827
|
+
return cowRuntime.formatUnmatchedChainOrderForLog(order);
|
|
2586
828
|
}
|
|
2587
829
|
/**
|
|
2588
830
|
* Record a pending CREATE broadcast on the manager.
|
|
@@ -2605,35 +847,7 @@ class DEXBot {
|
|
|
2605
847
|
* @returns {void}
|
|
2606
848
|
*/
|
|
2607
849
|
_recordPendingBroadcast(entry) {
|
|
2608
|
-
|
|
2609
|
-
return;
|
|
2610
|
-
if (!this.manager._pendingBroadcasts || !(this.manager._pendingBroadcasts instanceof Map)) {
|
|
2611
|
-
this.manager._pendingBroadcasts = new Map();
|
|
2612
|
-
}
|
|
2613
|
-
const fingerprint = buildCreateOpFingerprint({
|
|
2614
|
-
side: entry.order.type,
|
|
2615
|
-
assetA: this.manager?.assets?.assetA?.id,
|
|
2616
|
-
assetB: this.manager?.assets?.assetB?.id,
|
|
2617
|
-
sellInt: entry.finalInts?.sell,
|
|
2618
|
-
receiveInt: entry.finalInts?.receive,
|
|
2619
|
-
slotId: entry.order.id
|
|
2620
|
-
});
|
|
2621
|
-
if (!fingerprint) {
|
|
2622
|
-
this.manager.logger.log?.(`[COW] Skipped pending-broadcast record: could not build fingerprint for ${entry.order?.id || 'unknown'}`, 'warn');
|
|
2623
|
-
return;
|
|
2624
|
-
}
|
|
2625
|
-
this.manager._pendingBroadcasts.set(fingerprint, {
|
|
2626
|
-
fingerprint,
|
|
2627
|
-
opIndex: entry.opIndex,
|
|
2628
|
-
ctxIndex: entry.ctxIndex,
|
|
2629
|
-
slotId: entry.order.id,
|
|
2630
|
-
orderId: entry.order.id,
|
|
2631
|
-
orderType: entry.order.type,
|
|
2632
|
-
order: entry.order,
|
|
2633
|
-
finalInts: entry.finalInts,
|
|
2634
|
-
batchId: this._currentBatchId || null,
|
|
2635
|
-
recordedAt: Date.now()
|
|
2636
|
-
});
|
|
850
|
+
return cowRuntime.recordPendingBroadcast(this, entry);
|
|
2637
851
|
}
|
|
2638
852
|
/**
|
|
2639
853
|
* Clear the pending-broadcast cache.
|
|
@@ -2644,9 +858,7 @@ class DEXBot {
|
|
|
2644
858
|
* _reconcileAfterUncertainBroadcast before calling this).
|
|
2645
859
|
*/
|
|
2646
860
|
_clearPendingBroadcasts() {
|
|
2647
|
-
|
|
2648
|
-
this.manager._pendingBroadcasts.clear();
|
|
2649
|
-
}
|
|
861
|
+
return cowRuntime.clearPendingBroadcasts(this.manager?._pendingBroadcasts);
|
|
2650
862
|
}
|
|
2651
863
|
/**
|
|
2652
864
|
* Build a fingerprint for an on-chain order so it can be matched against
|
|
@@ -2657,19 +869,7 @@ class DEXBot {
|
|
|
2657
869
|
* @returns {string|null} Fingerprint or null on bad input
|
|
2658
870
|
*/
|
|
2659
871
|
_buildChainOrderFingerprint(chainOrder, slotId) {
|
|
2660
|
-
|
|
2661
|
-
return null;
|
|
2662
|
-
const normalized = this._normalizeChainOrderForPendingMatch(chainOrder);
|
|
2663
|
-
if (!normalized)
|
|
2664
|
-
return null;
|
|
2665
|
-
return buildCreateOpFingerprint({
|
|
2666
|
-
side: normalized.side,
|
|
2667
|
-
assetA: normalized.assetA,
|
|
2668
|
-
assetB: normalized.assetB,
|
|
2669
|
-
sellInt: normalized.sellInt,
|
|
2670
|
-
receiveInt: normalized.receiveInt,
|
|
2671
|
-
slotId
|
|
2672
|
-
});
|
|
872
|
+
return cowRuntime.buildChainOrderFingerprint(this, chainOrder, slotId);
|
|
2673
873
|
}
|
|
2674
874
|
/**
|
|
2675
875
|
* Normalize raw BitShares limit_order_object data into the integer tuple
|
|
@@ -2683,41 +883,7 @@ class DEXBot {
|
|
|
2683
883
|
* @returns {{side: string, assetA: string, assetB: string, sellInt: number, receiveInt: number}|null}
|
|
2684
884
|
*/
|
|
2685
885
|
_normalizeChainOrderForPendingMatch(chainOrder) {
|
|
2686
|
-
|
|
2687
|
-
return null;
|
|
2688
|
-
const assetA = this.manager?.assets?.assetA?.id;
|
|
2689
|
-
const assetB = this.manager?.assets?.assetB?.id;
|
|
2690
|
-
if (!assetA || !assetB)
|
|
2691
|
-
return null;
|
|
2692
|
-
const explicitSide = (chainOrder.type === 'buy' || chainOrder.type === 'sell')
|
|
2693
|
-
? chainOrder.type
|
|
2694
|
-
: null;
|
|
2695
|
-
const explicitSell = chainOrder.sellInt ?? chainOrder.sell;
|
|
2696
|
-
const explicitReceive = chainOrder.receiveInt ?? chainOrder.receive;
|
|
2697
|
-
if (explicitSide && Number.isFinite(Number(explicitSell)) && Number.isFinite(Number(explicitReceive))) {
|
|
2698
|
-
return {
|
|
2699
|
-
side: explicitSide,
|
|
2700
|
-
assetA,
|
|
2701
|
-
assetB,
|
|
2702
|
-
sellInt: Number(explicitSell),
|
|
2703
|
-
receiveInt: Number(explicitReceive)
|
|
2704
|
-
};
|
|
2705
|
-
}
|
|
2706
|
-
const base = chainOrder.sell_price?.base;
|
|
2707
|
-
const quote = chainOrder.sell_price?.quote;
|
|
2708
|
-
if (!base || !quote || !base.asset_id || !quote.asset_id)
|
|
2709
|
-
return null;
|
|
2710
|
-
const baseAmount = Number(base.amount);
|
|
2711
|
-
const quoteAmount = Number(quote.amount);
|
|
2712
|
-
if (!Number.isFinite(baseAmount) || !Number.isFinite(quoteAmount))
|
|
2713
|
-
return null;
|
|
2714
|
-
if (base.asset_id === assetA && quote.asset_id === assetB) {
|
|
2715
|
-
return { side: 'sell', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
|
|
2716
|
-
}
|
|
2717
|
-
if (base.asset_id === assetB && quote.asset_id === assetA) {
|
|
2718
|
-
return { side: 'buy', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
|
|
2719
|
-
}
|
|
2720
|
-
return null;
|
|
886
|
+
return cowRuntime.normalizeChainOrderForPendingMatch(this, chainOrder);
|
|
2721
887
|
}
|
|
2722
888
|
/**
|
|
2723
889
|
* Find a chain order that matches a planned slot using price+size proximity.
|
|
@@ -2735,56 +901,7 @@ class DEXBot {
|
|
|
2735
901
|
* @returns {Object|null} Matching chain order, or null
|
|
2736
902
|
*/
|
|
2737
903
|
_findChainOrderForSlot(chainOrders, slotId, planned) {
|
|
2738
|
-
|
|
2739
|
-
return null;
|
|
2740
|
-
const assetA = this.manager?.assets?.assetA?.id;
|
|
2741
|
-
const assetB = this.manager?.assets?.assetB?.id;
|
|
2742
|
-
if (!assetA || !assetB)
|
|
2743
|
-
return null;
|
|
2744
|
-
// 1. Exact fingerprint match.
|
|
2745
|
-
for (const o of chainOrders) {
|
|
2746
|
-
const fp = this._buildChainOrderFingerprint(o, slotId);
|
|
2747
|
-
if (fp && this.manager._pendingBroadcasts?.has(fp)) {
|
|
2748
|
-
return o;
|
|
2749
|
-
}
|
|
2750
|
-
}
|
|
2751
|
-
if (!planned || !Number.isFinite(Number(planned.sell)) || !Number.isFinite(Number(planned.receive))) {
|
|
2752
|
-
return null;
|
|
2753
|
-
}
|
|
2754
|
-
// 2. Near match: same side, sell int within 1, receive int within 1% or 2 units.
|
|
2755
|
-
const targetSell = Number(planned.sell);
|
|
2756
|
-
const targetReceive = Number(planned.receive);
|
|
2757
|
-
const plannedSide = planned.orderType ||
|
|
2758
|
-
planned.side ||
|
|
2759
|
-
this.manager._pendingBroadcasts?.get?.(planned.fingerprint)?.orderType ||
|
|
2760
|
-
this.manager.orders.get(slotId)?.type;
|
|
2761
|
-
if (plannedSide !== 'buy' && plannedSide !== 'sell') {
|
|
2762
|
-
return null;
|
|
2763
|
-
}
|
|
2764
|
-
let best = null;
|
|
2765
|
-
let bestDistance = Infinity;
|
|
2766
|
-
for (const o of chainOrders) {
|
|
2767
|
-
const normalized = this._normalizeChainOrderForPendingMatch(o);
|
|
2768
|
-
if (!normalized)
|
|
2769
|
-
continue;
|
|
2770
|
-
if (normalized.side !== plannedSide)
|
|
2771
|
-
continue;
|
|
2772
|
-
const sell = Number(normalized.sellInt);
|
|
2773
|
-
const receive = Number(normalized.receiveInt);
|
|
2774
|
-
if (!Number.isFinite(sell) || !Number.isFinite(receive))
|
|
2775
|
-
continue;
|
|
2776
|
-
const sellDelta = Math.abs(sell - targetSell);
|
|
2777
|
-
const receiveDelta = Math.abs(receive - targetReceive);
|
|
2778
|
-
const receiveTol = Math.max(2, Math.floor(targetReceive * 0.01));
|
|
2779
|
-
if (sellDelta > 1 || receiveDelta > receiveTol)
|
|
2780
|
-
continue;
|
|
2781
|
-
const distance = sellDelta * 1000 + receiveDelta;
|
|
2782
|
-
if (distance < bestDistance) {
|
|
2783
|
-
best = o;
|
|
2784
|
-
bestDistance = distance;
|
|
2785
|
-
}
|
|
2786
|
-
}
|
|
2787
|
-
return best;
|
|
904
|
+
return cowRuntime.findChainOrderForSlot(this, chainOrders, slotId, planned);
|
|
2788
905
|
}
|
|
2789
906
|
/**
|
|
2790
907
|
* Reconcile a broadcast whose chain state is unknown.
|
|
@@ -2799,287 +916,24 @@ class DEXBot {
|
|
|
2799
916
|
* 2. For each pending-broadcast entry (fingerprinted CREATE op), look
|
|
2800
917
|
* for a matching chain order. If found: adopt it (set the opContext's
|
|
2801
918
|
* chainOrderId and continue with the existing planned slot).
|
|
2802
|
-
* 3. For pending entries with no chain match: virtualize (mark the
|
|
2803
|
-
* opContext as discarded; the planned slot stays empty until the
|
|
2804
|
-
* next planning cycle).
|
|
2805
|
-
* 4. Persist + log a structured [COW][UNCERTAIN] summary.
|
|
2806
|
-
*
|
|
2807
|
-
* After this method returns, the bot has either adopted the on-chain
|
|
2808
|
-
* result (good case — chain accepted but we didn't see the reply) or
|
|
2809
|
-
* accepted the discard (chain rejected or never received the op).
|
|
2810
|
-
*
|
|
2811
|
-
* @param {BroadcastUncertainError} err - The thrown error
|
|
2812
|
-
* @param {Array<Object>} opContexts - Original opContexts from the failed batch
|
|
2813
|
-
* @returns {Promise<Object>} Result object compatible with batch return shape
|
|
2814
|
-
*/
|
|
2815
|
-
async _reconcileAfterUncertainBroadcast(err, opContexts, options = {}) {
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
}
|
|
2821
|
-
return this._reconcileAfterUncertainBroadcastImpl(err, opContexts, options);
|
|
2822
|
-
}
|
|
2823
|
-
async _reconcileAfterUncertainBroadcastImpl(err, opContexts, options) {
|
|
2824
|
-
const startedAt = Date.now();
|
|
2825
|
-
const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
|
|
2826
|
-
? Array.from(this.manager._pendingBroadcasts.values())
|
|
2827
|
-
: [];
|
|
2828
|
-
const createContextCount = opContexts.filter(c => c && c.kind === 'create').length;
|
|
2829
|
-
const nonCreateContextCount = opContexts.length - createContextCount;
|
|
2830
|
-
this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
|
|
2831
|
-
`creates=${createContextCount} nonCreates=${nonCreateContextCount} ` +
|
|
2832
|
-
`staleSinceMs=${err?.timeoutMs || 'n/a'}. Entering reconcile-then-decide.`, 'warn');
|
|
2833
|
-
if (!chainOrders?.readOpenOrders) {
|
|
2834
|
-
this.manager.logger.log('[COW][UNCERTAIN] readOpenOrders unavailable; falling back to structural resync only.', 'error');
|
|
2835
|
-
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
2836
|
-
await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders unavailable', { batchId: err?.batchId || null });
|
|
2837
|
-
}
|
|
2838
|
-
this._clearPendingBroadcasts();
|
|
2839
|
-
return { executed: false, hadRotation: false, uncertain: true };
|
|
2840
|
-
}
|
|
2841
|
-
// 1. Read the chain
|
|
2842
|
-
const accountRef = this.accountId || this.account?.id || this.account;
|
|
2843
|
-
let chainSnapshot = [];
|
|
2844
|
-
try {
|
|
2845
|
-
chainSnapshot = await chainOrders.readOpenOrders(accountRef);
|
|
2846
|
-
}
|
|
2847
|
-
catch (readErr) {
|
|
2848
|
-
this.manager.logger.log(`[COW][UNCERTAIN] readOpenOrders failed: ${readErr?.message || readErr}. ` +
|
|
2849
|
-
`Falling back to structural resync.`, 'error');
|
|
2850
|
-
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
2851
|
-
await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders failed', { batchId: err?.batchId || null, error: readErr?.message || String(readErr) });
|
|
2852
|
-
}
|
|
2853
|
-
this._clearPendingBroadcasts();
|
|
2854
|
-
return { executed: false, hadRotation: false, uncertain: true };
|
|
2855
|
-
}
|
|
2856
|
-
const adopted = [];
|
|
2857
|
-
let discarded = [];
|
|
2858
|
-
// 2. For each pending broadcast, look for a chain match.
|
|
2859
|
-
for (const entry of pending) {
|
|
2860
|
-
const match = this._findChainOrderForSlot(chainSnapshot, entry.slotId, {
|
|
2861
|
-
sell: entry.finalInts?.sell,
|
|
2862
|
-
receive: entry.finalInts?.receive,
|
|
2863
|
-
orderType: entry.orderType || entry.order?.type,
|
|
2864
|
-
fingerprint: entry.fingerprint
|
|
2865
|
-
});
|
|
2866
|
-
if (match) {
|
|
2867
|
-
adopted.push({
|
|
2868
|
-
slotId: entry.slotId,
|
|
2869
|
-
chainOrderId: match.id,
|
|
2870
|
-
orderType: entry.orderType || entry.order?.type,
|
|
2871
|
-
});
|
|
2872
|
-
this.manager._pendingBroadcasts.delete(entry.fingerprint);
|
|
2873
|
-
}
|
|
2874
|
-
else {
|
|
2875
|
-
// Preserve full entry context so the recheck below can use
|
|
2876
|
-
// both the fingerprint match path (against _pendingBroadcasts)
|
|
2877
|
-
// and the near-match path (via finalInts/orderType), not just
|
|
2878
|
-
// slotId alone.
|
|
2879
|
-
discarded.push(entry);
|
|
2880
|
-
}
|
|
2881
|
-
}
|
|
2882
|
-
// ---- RACE MITIGATION: block-delay recheck before discarding creates ----
|
|
2883
|
-
// BROADCAST_DEADLINE fires while the transaction may still be valid
|
|
2884
|
-
// and land in the next block. Wait for up to 3 blocks (~1 BitShares
|
|
2885
|
-
// block interval each) and re-check before permanently discarding.
|
|
2886
|
-
const UNCERTAIN_RECHECK_MAX_ATTEMPTS = this.config.pipelineTiming?.RETRY_MAX_ATTEMPTS ?? PIPELINE_TIMING.RETRY_MAX_ATTEMPTS; // 3
|
|
2887
|
-
const UNCERTAIN_RECHECK_INTERVAL_MS = TIMING.MILLISECONDS_PER_SECOND * 3; // 3s
|
|
2888
|
-
let recheckRound = 0;
|
|
2889
|
-
while (discarded.length > 0 && recheckRound < UNCERTAIN_RECHECK_MAX_ATTEMPTS) {
|
|
2890
|
-
recheckRound++;
|
|
2891
|
-
await new Promise(r => setTimeout(r, UNCERTAIN_RECHECK_INTERVAL_MS));
|
|
2892
|
-
let freshSnapshot;
|
|
2893
|
-
try {
|
|
2894
|
-
freshSnapshot = await chainOrders.readOpenOrders(accountRef);
|
|
2895
|
-
}
|
|
2896
|
-
catch {
|
|
2897
|
-
break;
|
|
2898
|
-
}
|
|
2899
|
-
const stillDiscarded = [];
|
|
2900
|
-
for (const entry of discarded) {
|
|
2901
|
-
const match = this._findChainOrderForSlot(freshSnapshot, entry.slotId, {
|
|
2902
|
-
sell: entry.finalInts?.sell,
|
|
2903
|
-
receive: entry.finalInts?.receive,
|
|
2904
|
-
orderType: entry.orderType || entry.order?.type,
|
|
2905
|
-
fingerprint: entry.fingerprint
|
|
2906
|
-
});
|
|
2907
|
-
if (match) {
|
|
2908
|
-
adopted.push({
|
|
2909
|
-
slotId: entry.slotId,
|
|
2910
|
-
chainOrderId: match.id,
|
|
2911
|
-
orderType: entry.orderType || entry.order?.type,
|
|
2912
|
-
});
|
|
2913
|
-
this.manager.logger.log(`[COW][UNCERTAIN] CREATE re-adopted after ${recheckRound} block(s): ` +
|
|
2914
|
-
`${entry.slotId}->${match.id}`, 'info');
|
|
2915
|
-
}
|
|
2916
|
-
else {
|
|
2917
|
-
stillDiscarded.push(entry);
|
|
2918
|
-
}
|
|
2919
|
-
}
|
|
2920
|
-
discarded = stillDiscarded;
|
|
2921
|
-
}
|
|
2922
|
-
// 3. Re-read chain with latest state. The initial snapshot may be
|
|
2923
|
-
// stale for non-CREATE ops (updates/cancels) not tracked in
|
|
2924
|
-
// _pendingBroadcasts — the seed cause of the slot-115 cascade.
|
|
2925
|
-
let latestSnapshot;
|
|
2926
|
-
try {
|
|
2927
|
-
latestSnapshot = await chainOrders.readOpenOrders(accountRef);
|
|
2928
|
-
}
|
|
2929
|
-
catch {
|
|
2930
|
-
latestSnapshot = chainSnapshot;
|
|
2931
|
-
}
|
|
2932
|
-
// Always sync to link adopted chain orders into grid slots.
|
|
2933
|
-
// Otherwise an adopted slot stays VIRTUAL and the next COW cycle
|
|
2934
|
-
// places a duplicate order at the same price.
|
|
2935
|
-
let hadRotation = false;
|
|
2936
|
-
if (latestSnapshot && latestSnapshot.length > 0 && this.manager?.syncFromOpenOrders) {
|
|
2937
|
-
try {
|
|
2938
|
-
await this.manager.syncFromOpenOrders(latestSnapshot, {
|
|
2939
|
-
skipAccounting: true,
|
|
2940
|
-
});
|
|
2941
|
-
hadRotation = true;
|
|
2942
|
-
}
|
|
2943
|
-
catch (syncErr) {
|
|
2944
|
-
this.manager.logger.log(`[COW][UNCERTAIN] syncFromOpenOrders failed during recovery: ${syncErr?.message || syncErr}`, 'error');
|
|
2945
|
-
// Fallback: reload from persisted snapshot + re-sync.
|
|
2946
|
-
this.manager.logger.log('[COW][UNCERTAIN] syncFromOpenOrders failed; falling back to full grid reload from persisted snapshot', 'warn');
|
|
2947
|
-
const fallbackResult = await this._recoverFromPersistedGrid();
|
|
2948
|
-
if (fallbackResult.success) {
|
|
2949
|
-
hadRotation = true;
|
|
2950
|
-
}
|
|
2951
|
-
else {
|
|
2952
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Persisted grid reload failed: ${fallbackResult.reason}. ` +
|
|
2953
|
-
`Escalating to structural resync.`, 'warn');
|
|
2954
|
-
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
2955
|
-
this.manager.requestStructuralGridResync('cow-uncertain-recovery-failed', { reason: fallbackResult.reason }).catch((escalateErr) => {
|
|
2956
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Escalation to structural resync failed: ${escalateErr.message}`, 'error');
|
|
2957
|
-
});
|
|
2958
|
-
}
|
|
2959
|
-
}
|
|
2960
|
-
}
|
|
2961
|
-
}
|
|
2962
|
-
// 4. Log structured summary.
|
|
2963
|
-
const elapsedMs = Date.now() - startedAt;
|
|
2964
|
-
const heartbeatAgeMs = this._lastBroadcastHeartbeatAt
|
|
2965
|
-
? Date.now() - this._lastBroadcastHeartbeatAt
|
|
2966
|
-
: null;
|
|
2967
|
-
this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
|
|
2968
|
-
`staleSinceMs=${err?.timeoutMs || 'n/a'} heartbeatAgeMs=${heartbeatAgeMs ?? 'n/a'} ` +
|
|
2969
|
-
`adopted=${adopted.length} discarded=${discarded.length} recheckRounds=${recheckRound} elapsedMs=${elapsedMs}`, adopted.length > 0 ? 'info' : 'warn');
|
|
2970
|
-
if (adopted.length > 0) {
|
|
2971
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Adopted chain orders: ${adopted
|
|
2972
|
-
.map(a => `${a.slotId}->${a.chainOrderId}`)
|
|
2973
|
-
.join(', ')}`, 'info');
|
|
2974
|
-
}
|
|
2975
|
-
if (discarded.length > 0) {
|
|
2976
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Discarded planned CREATEs (no chain match after ${recheckRound} recheck(s)): ${discarded
|
|
2977
|
-
.map(d => d.slotId)
|
|
2978
|
-
.join(', ')}`, 'warn');
|
|
2979
|
-
// Restore target grid sizes for discarded CREATEs so the slots are
|
|
2980
|
-
// not left as virtual/0 after recovery. Without this, the slot stays
|
|
2981
|
-
// at size 0 (set by the fill handler) and is never reactivated until
|
|
2982
|
-
// a fresh fill cycle triggers another rebalance that happens to succeed.
|
|
2983
|
-
// entry.order is the pending-broadcast target order (captured at broadcast
|
|
2984
|
-
// time); its .id matches entry.slotId — the lookup below keys on slotId.
|
|
2985
|
-
for (const entry of discarded) {
|
|
2986
|
-
if (entry.order && entry.slotId) {
|
|
2987
|
-
const current = this.manager.orders.get(entry.slotId);
|
|
2988
|
-
if (current && current.state === ORDER_STATES.VIRTUAL && !current.orderId) {
|
|
2989
|
-
try {
|
|
2990
|
-
await this.manager._applyOrderUpdate({ ...entry.order, state: ORDER_STATES.VIRTUAL, orderId: null }, 'uncertain-recovery-restore-size', { skipAccounting: true, fee: 0 });
|
|
2991
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Restored target size for discarded CREATE slot ${entry.slotId} (size: ${entry.order.size})`, 'info');
|
|
2992
|
-
}
|
|
2993
|
-
catch (restoreErr) {
|
|
2994
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Failed to restore size for slot ${entry.slotId}: ${restoreErr?.message || restoreErr}`, 'warn');
|
|
2995
|
-
}
|
|
2996
|
-
}
|
|
2997
|
-
}
|
|
2998
|
-
}
|
|
2999
|
-
}
|
|
3000
|
-
// ---- Structural resync safeguard after skipAccounting restore ----
|
|
3001
|
-
// The discarded CREATE restore above used skipAccounting: true to avoid
|
|
3002
|
-
// double-counting when the structural resync recalculates accounting
|
|
3003
|
-
// from scratch. Ensure one is scheduled — if already in-flight (timer
|
|
3004
|
-
// or running), the existing resync will handle the accounting fix.
|
|
3005
|
-
if (discarded.length > 0 && typeof this.manager?.requestStructuralGridResync === 'function') {
|
|
3006
|
-
const alreadyScheduled = this._structuralGridResyncRunning || this._structuralGridResyncTimer;
|
|
3007
|
-
if (!alreadyScheduled) {
|
|
3008
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Scheduling structural resync after discarded CREATE restore ` +
|
|
3009
|
-
`(skipAccounting used — resync needed for fund recalculation)`, 'info');
|
|
3010
|
-
this.manager.requestStructuralGridResync('cow-uncertain-accounting-repair', { discardedCount: discarded.length }).catch((err) => {
|
|
3011
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Failed to schedule accounting repair resync: ${err.message}`, 'error');
|
|
3012
|
-
});
|
|
3013
|
-
}
|
|
3014
|
-
else {
|
|
3015
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Structural resync already ${this._structuralGridResyncRunning ? 'running' : 'scheduled'}; ` +
|
|
3016
|
-
`it will repair accounting after discarded CREATE restore`, 'debug');
|
|
3017
|
-
}
|
|
3018
|
-
}
|
|
3019
|
-
this._clearPendingBroadcasts();
|
|
3020
|
-
// Post-recovery safety net: if there are still unmatched chain
|
|
3021
|
-
// orders after reconcile-then-decide, cancel ONE per cycle. The cap
|
|
3022
|
-
// is enforced inside the helper via _autoCancelOrphanCycleMarker.
|
|
3023
|
-
try {
|
|
3024
|
-
const autoCancelResult = await this._autoCancelOneUnmatchedOrphan();
|
|
3025
|
-
if (autoCancelResult.cancelled) {
|
|
3026
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancelled orphan ${autoCancelResult.orderId} ` +
|
|
3027
|
-
`after recovery (cap=1/cycle).`, 'info');
|
|
3028
|
-
}
|
|
3029
|
-
}
|
|
3030
|
-
catch (orphanErr) {
|
|
3031
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancel pass failed: ${orphanErr?.message || orphanErr}`, 'warn');
|
|
3032
|
-
}
|
|
3033
|
-
// Boundary shift recovery: the COW batch that should have committed the
|
|
3034
|
-
// boundary shift from this fill cycle failed before commit. Only COUNT
|
|
3035
|
-
// ADOPTED CREATEs — orders that actually landed on-chain. Discarded
|
|
3036
|
-
// CREATEs represent orders that never existed, so the grid must NOT
|
|
3037
|
-
// shift the boundary for them. The next fill cycle's
|
|
3038
|
-
// calculateTargetGrid → deriveTargetBoundary will recompute the correct
|
|
3039
|
-
// boundary from scratch for any discarded slots.
|
|
3040
|
-
//
|
|
3041
|
-
// NOTE: Direct mutation of manager.boundaryIdx outside a COW commit
|
|
3042
|
-
// violates the invariant stated in grid.ts:1217-1220 (boundary must only
|
|
3043
|
-
// be updated atomically inside _commitWorkingGrid). This is intentional
|
|
3044
|
-
// here because the COW commit already failed — the invariant was already
|
|
3045
|
-
// broken by the broadcast uncertainty, and the adjustment only restores
|
|
3046
|
-
// the state to what the successful commit would have produced for
|
|
3047
|
-
// orders that are confirmed on-chain.
|
|
3048
|
-
if (hadRotation && this.manager && adopted.length > 0) {
|
|
3049
|
-
let boundaryShift = 0;
|
|
3050
|
-
for (const entry of adopted) {
|
|
3051
|
-
if (entry.orderType === ORDER_TYPES.SELL) {
|
|
3052
|
-
boundaryShift--; // SELL CREATE → fill was BUY → boundary LEFT
|
|
3053
|
-
}
|
|
3054
|
-
else if (entry.orderType === ORDER_TYPES.BUY) {
|
|
3055
|
-
boundaryShift++; // BUY CREATE → fill was SELL → boundary RIGHT
|
|
3056
|
-
}
|
|
3057
|
-
}
|
|
3058
|
-
if (boundaryShift !== 0) {
|
|
3059
|
-
const oldIdx = this.manager.boundaryIdx;
|
|
3060
|
-
if (typeof oldIdx === 'number') {
|
|
3061
|
-
const maxIdx = Math.max(0, (this.manager.orders?.size || 1) - 1);
|
|
3062
|
-
const newIdx = Math.max(0, Math.min(maxIdx, oldIdx + boundaryShift));
|
|
3063
|
-
if (newIdx !== oldIdx) {
|
|
3064
|
-
this.manager.boundaryIdx = newIdx;
|
|
3065
|
-
this.manager.logger.log(`[COW][UNCERTAIN] Boundary adjusted by ${boundaryShift} ` +
|
|
3066
|
-
`(${adopted.length} adopted CREATE(s), ${discarded.length} discarded): ` +
|
|
3067
|
-
`${oldIdx} → ${newIdx}. Next COW cycle will generate replacement opposite-side orders.`, 'warn');
|
|
3068
|
-
if (typeof this.manager._markGridDirty === 'function') {
|
|
3069
|
-
this.manager._markGridDirty();
|
|
3070
|
-
}
|
|
3071
|
-
}
|
|
3072
|
-
}
|
|
3073
|
-
}
|
|
3074
|
-
}
|
|
3075
|
-
// Persist master grid mutations from the reconciliation sync and any
|
|
3076
|
-
// orphan auto-cancellation that ran above. These apply outside the COW
|
|
3077
|
-
// broadcast path and would otherwise be in-memory only. The boundary
|
|
3078
|
-
// adjustment above is also persisted here.
|
|
3079
|
-
if (typeof this.manager?.persistGrid === 'function') {
|
|
3080
|
-
await this.manager.persistGrid();
|
|
3081
|
-
}
|
|
3082
|
-
return { executed: false, hadRotation, uncertain: true, adopted, discarded };
|
|
919
|
+
* 3. For pending entries with no chain match: virtualize (mark the
|
|
920
|
+
* opContext as discarded; the planned slot stays empty until the
|
|
921
|
+
* next planning cycle).
|
|
922
|
+
* 4. Persist + log a structured [COW][UNCERTAIN] summary.
|
|
923
|
+
*
|
|
924
|
+
* After this method returns, the bot has either adopted the on-chain
|
|
925
|
+
* result (good case — chain accepted but we didn't see the reply) or
|
|
926
|
+
* accepted the discard (chain rejected or never received the op).
|
|
927
|
+
*
|
|
928
|
+
* @param {BroadcastUncertainError} err - The thrown error
|
|
929
|
+
* @param {Array<Object>} opContexts - Original opContexts from the failed batch
|
|
930
|
+
* @returns {Promise<Object>} Result object compatible with batch return shape
|
|
931
|
+
*/
|
|
932
|
+
async _reconcileAfterUncertainBroadcast(err, opContexts, options = {}) {
|
|
933
|
+
return cowRuntime.reconcileAfterUncertainBroadcast(this, err, opContexts, options);
|
|
934
|
+
}
|
|
935
|
+
async _reconcileAfterUncertainBroadcastImpl(err, opContexts, options) {
|
|
936
|
+
return cowRuntime.reconcileAfterUncertainBroadcastImpl(this, err, opContexts, options);
|
|
3083
937
|
}
|
|
3084
938
|
/**
|
|
3085
939
|
* Auto-cancel a price-drift orphan from the unmatched-order snapshot.
|
|
@@ -3107,72 +961,7 @@ class DEXBot {
|
|
|
3107
961
|
* @returns {Promise<{cancelled: boolean, orderId?: string, reason?: string}>}
|
|
3108
962
|
*/
|
|
3109
963
|
async _autoCancelOneUnmatchedOrphan() {
|
|
3110
|
-
|
|
3111
|
-
// FIX 4: Higher cap during recovery mode so orphan backlog clears faster.
|
|
3112
|
-
const recoveryActive = this.manager?._recoveryState?.structuralResyncRequested === true;
|
|
3113
|
-
const cycleCap = recoveryActive ? 5 : 1;
|
|
3114
|
-
if (this._autoCancelOrphanCycleMarker === cycleId) {
|
|
3115
|
-
if (this._autoCancelOrphanSubCount >= cycleCap) {
|
|
3116
|
-
return { cancelled: false, reason: 'cap-reached-this-cycle', subCount: this._autoCancelOrphanSubCount };
|
|
3117
|
-
}
|
|
3118
|
-
}
|
|
3119
|
-
else {
|
|
3120
|
-
// Reset sub-count on first call this cycle
|
|
3121
|
-
this._autoCancelOrphanCycleMarker = cycleId;
|
|
3122
|
-
this._autoCancelOrphanSubCount = 0;
|
|
3123
|
-
}
|
|
3124
|
-
const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
|
|
3125
|
-
? this.manager._pendingBroadcasts.size
|
|
3126
|
-
: 0;
|
|
3127
|
-
if (pending > 0) {
|
|
3128
|
-
return { cancelled: false, reason: 'pending-broadcasts-active' };
|
|
3129
|
-
}
|
|
3130
|
-
const unmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
|
|
3131
|
-
? this.manager._lastUnmatchedChainOrders
|
|
3132
|
-
: [];
|
|
3133
|
-
if (unmatched.length === 0) {
|
|
3134
|
-
return { cancelled: false, reason: 'no-unmatched' };
|
|
3135
|
-
}
|
|
3136
|
-
// Check for fingerprinted entries first — these came from a pending
|
|
3137
|
-
// broadcast (missing-create-result) and must be handled by the recovery
|
|
3138
|
-
// path, not by auto-cancel. The first fingerprinted entry is checked
|
|
3139
|
-
// regardless of its reason field.
|
|
3140
|
-
const fingerprinted = unmatched.find(u => u && u.fingerprint);
|
|
3141
|
-
if (fingerprinted) {
|
|
3142
|
-
return { cancelled: false, reason: 'fingerprinted-handle-via-recovery' };
|
|
3143
|
-
}
|
|
3144
|
-
// Only cancel price-drift orphans — these are surplus orders that
|
|
3145
|
-
// drifted away from their assigned slot price and have no adoptable
|
|
3146
|
-
// grid slot. All other unmatched orders (duplicate-price-level,
|
|
3147
|
-
// already-matched-slot, etc.) are adoptable positions that the structural
|
|
3148
|
-
// resync will integrate into the grid — cancelling them destroys capital.
|
|
3149
|
-
const target = unmatched.find(u => u && u.reason === 'price-drift-orphan');
|
|
3150
|
-
if (!target) {
|
|
3151
|
-
return { cancelled: false, reason: 'no-price-drift-orphan', message: 'no price-drift orphan to cancel; other unmatched orders are adoptable' };
|
|
3152
|
-
}
|
|
3153
|
-
const orderId = target.id || target.orderId || target.chainOrderId;
|
|
3154
|
-
if (!orderId) {
|
|
3155
|
-
return { cancelled: false, reason: 'no-orderId' };
|
|
3156
|
-
}
|
|
3157
|
-
if (!chainOrders?.cancelOrder) {
|
|
3158
|
-
return { cancelled: false, reason: 'cancelOrder-unavailable' };
|
|
3159
|
-
}
|
|
3160
|
-
try {
|
|
3161
|
-
await chainOrders.cancelOrder(this.account, this.privateKey, orderId);
|
|
3162
|
-
if (typeof chainOrders.recordOwnCancel === 'function') {
|
|
3163
|
-
chainOrders.recordOwnCancel(orderId);
|
|
3164
|
-
}
|
|
3165
|
-
// Increment sub-count only after a confirmed successful cancel.
|
|
3166
|
-
// Failed cancels do not consume the per-cycle cap budget.
|
|
3167
|
-
this._autoCancelOrphanSubCount++;
|
|
3168
|
-
this.manager.logger.log(`[COW] Auto-cancelled ${this._autoCancelOrphanSubCount}/${unmatched.length} unmatched chain order ` +
|
|
3169
|
-
`(${this._formatUnmatchedChainOrderForLog(target)}) — per-cycle cap=${cycleCap}.`, 'warn');
|
|
3170
|
-
return { cancelled: true, orderId };
|
|
3171
|
-
}
|
|
3172
|
-
catch (err) {
|
|
3173
|
-
this.manager.logger.log(`[COW] Auto-cancel of unmatched chain order ${orderId} failed: ${err?.message || err}`, 'error');
|
|
3174
|
-
return { cancelled: false, reason: 'cancel-failed', error: err?.message || String(err) };
|
|
3175
|
-
}
|
|
964
|
+
return cowRuntime.autoCancelOneUnmatchedOrphan(this);
|
|
3176
965
|
}
|
|
3177
966
|
// Pair mode applies only when create contexts include both BUY and SELL.
|
|
3178
967
|
// Single-side create batches intentionally remain a single executeBatch.
|
|
@@ -3182,21 +971,7 @@ class DEXBot {
|
|
|
3182
971
|
* @returns {boolean} True if pair mode should be used
|
|
3183
972
|
*/
|
|
3184
973
|
_shouldExecuteCreatePairMode(opContexts) {
|
|
3185
|
-
|
|
3186
|
-
return false;
|
|
3187
|
-
if (!opContexts.every(ctx => ctx?.kind === 'create' && ctx?.order))
|
|
3188
|
-
return false;
|
|
3189
|
-
let hasBuy = false;
|
|
3190
|
-
let hasSell = false;
|
|
3191
|
-
for (const ctx of opContexts) {
|
|
3192
|
-
if (ctx.order.type === ORDER_TYPES.BUY)
|
|
3193
|
-
hasBuy = true;
|
|
3194
|
-
if (ctx.order.type === ORDER_TYPES.SELL)
|
|
3195
|
-
hasSell = true;
|
|
3196
|
-
if (hasBuy && hasSell)
|
|
3197
|
-
return true;
|
|
3198
|
-
}
|
|
3199
|
-
return false;
|
|
974
|
+
return cowRuntime.shouldExecuteCreatePairMode(this, opContexts);
|
|
3200
975
|
}
|
|
3201
976
|
/**
|
|
3202
977
|
* Execute operations with retry on BroadcastUncertainError.
|
|
@@ -3208,36 +983,7 @@ class DEXBot {
|
|
|
3208
983
|
* the full operations array would duplicate those creates on chain.
|
|
3209
984
|
*/
|
|
3210
985
|
async _executeWithRetryOnUncertain(operations, opContexts) {
|
|
3211
|
-
|
|
3212
|
-
for (let attempt = 1;; attempt++) {
|
|
3213
|
-
try {
|
|
3214
|
-
return await this._executeOperationsWithStrategy(operations, opContexts);
|
|
3215
|
-
}
|
|
3216
|
-
catch (err) {
|
|
3217
|
-
const isRetriable = err instanceof BroadcastUncertainError
|
|
3218
|
-
&& !err.partialOnChainState
|
|
3219
|
-
&& attempt <= MAX_RETRIES;
|
|
3220
|
-
if (isRetriable) {
|
|
3221
|
-
this.manager.logger.log(`[COW] Broadcast uncertain (attempt ${attempt}/${MAX_RETRIES + 1}), retrying...`, 'warn');
|
|
3222
|
-
await this._ensureCredentialDaemonWritable('COW batch retry');
|
|
3223
|
-
// Reconcile before retry — stale ops would fail "not found".
|
|
3224
|
-
try {
|
|
3225
|
-
const accountRef = this.accountId || this.account?.id || this.account;
|
|
3226
|
-
const freshChain = await chainOrders.readOpenOrders(accountRef);
|
|
3227
|
-
if (freshChain.length > 0 && this.manager?.syncFromOpenOrders) {
|
|
3228
|
-
await this.manager.syncFromOpenOrders(freshChain, {
|
|
3229
|
-
skipAccounting: true,
|
|
3230
|
-
});
|
|
3231
|
-
}
|
|
3232
|
-
}
|
|
3233
|
-
catch (syncErr) {
|
|
3234
|
-
this.manager.logger.log(`[COW] Pre-retry sync failed (non-fatal): ${syncErr?.message || syncErr}`, 'warn');
|
|
3235
|
-
}
|
|
3236
|
-
continue;
|
|
3237
|
-
}
|
|
3238
|
-
throw err;
|
|
3239
|
-
}
|
|
3240
|
-
}
|
|
986
|
+
return cowRuntime.executeWithRetryOnUncertain(this, operations, opContexts);
|
|
3241
987
|
}
|
|
3242
988
|
/**
|
|
3243
989
|
* Execute blockchain operations with appropriate strategy (single batch or pair mode).
|
|
@@ -3246,63 +992,7 @@ class DEXBot {
|
|
|
3246
992
|
* @returns {Promise<{result: Object, opContexts: Array}>} Execution result with contexts
|
|
3247
993
|
*/
|
|
3248
994
|
async _executeOperationsWithStrategy(operations, opContexts) {
|
|
3249
|
-
|
|
3250
|
-
const result = await chainOrders.executeBatch(this.account, this.privateKey, operations);
|
|
3251
|
-
return { result, opContexts };
|
|
3252
|
-
}
|
|
3253
|
-
const createEntries = [];
|
|
3254
|
-
for (let i = 0; i < operations.length; i++) {
|
|
3255
|
-
createEntries.push({
|
|
3256
|
-
operation: operations[i],
|
|
3257
|
-
context: opContexts[i],
|
|
3258
|
-
});
|
|
3259
|
-
}
|
|
3260
|
-
const groups = this._buildOutsideInPairGroupsForCreateEntries(createEntries);
|
|
3261
|
-
const mergedOperationResults = [];
|
|
3262
|
-
const mergedRawResults = [];
|
|
3263
|
-
const mergedContexts = [];
|
|
3264
|
-
// Grouped execution is fail-fast, but NOT atomic across groups.
|
|
3265
|
-
// Each group is a separate on-chain transaction; if a later group fails,
|
|
3266
|
-
// earlier groups may already be confirmed on-chain.
|
|
3267
|
-
for (let idx = 0; idx < groups.length; idx++) {
|
|
3268
|
-
const group = groups[idx];
|
|
3269
|
-
const groupOps = group.map(e => e.operation);
|
|
3270
|
-
const groupContexts = group.map(e => e.context);
|
|
3271
|
-
this.manager.logger.log(`[COW] Broadcasting create pair group ${idx + 1}/${groups.length} (${groupOps.length} op${groupOps.length > 1 ? 's' : ''}, outside->center)`, 'info');
|
|
3272
|
-
let groupResult;
|
|
3273
|
-
try {
|
|
3274
|
-
groupResult = await chainOrders.executeBatch(this.account, this.privateKey, groupOps);
|
|
3275
|
-
}
|
|
3276
|
-
catch (err) {
|
|
3277
|
-
const groupsBroadcast = idx;
|
|
3278
|
-
const groupsTotal = groups.length;
|
|
3279
|
-
const broadcastedOperationCount = mergedContexts.length;
|
|
3280
|
-
this.manager.logger.log(`[COW] Grouped create execution failed at group ${idx + 1}/${groupsTotal}; ${groupsBroadcast} group(s) already broadcast (${broadcastedOperationCount} op context(s)). Partial on-chain state is possible.`, 'error');
|
|
3281
|
-
err.partialOnChainState = groupsBroadcast > 0;
|
|
3282
|
-
err.groupsBroadcast = groupsBroadcast;
|
|
3283
|
-
err.groupsTotal = groupsTotal;
|
|
3284
|
-
err.broadcastedOperationCount = broadcastedOperationCount;
|
|
3285
|
-
throw err;
|
|
3286
|
-
}
|
|
3287
|
-
const groupOpResults = this._extractOperationResults(groupResult);
|
|
3288
|
-
mergedOperationResults.push(...groupOpResults);
|
|
3289
|
-
mergedRawResults.push(groupResult?.raw || null);
|
|
3290
|
-
mergedContexts.push(...groupContexts);
|
|
3291
|
-
}
|
|
3292
|
-
return {
|
|
3293
|
-
result: {
|
|
3294
|
-
success: true,
|
|
3295
|
-
raw: {
|
|
3296
|
-
grouped: true,
|
|
3297
|
-
groupsExecuted: groups.length,
|
|
3298
|
-
groupResults: mergedRawResults,
|
|
3299
|
-
},
|
|
3300
|
-
operation_results: mergedOperationResults,
|
|
3301
|
-
grouped: true,
|
|
3302
|
-
groupsExecuted: groups.length
|
|
3303
|
-
},
|
|
3304
|
-
opContexts: mergedContexts
|
|
3305
|
-
};
|
|
995
|
+
return cowRuntime.executeOperationsWithStrategy(this, operations, opContexts);
|
|
3306
996
|
}
|
|
3307
997
|
/**
|
|
3308
998
|
* Validate that operations can be executed with available funds before broadcasting.
|
|
@@ -3314,94 +1004,7 @@ class DEXBot {
|
|
|
3314
1004
|
* @private
|
|
3315
1005
|
*/
|
|
3316
1006
|
_validateOperationFunds(operations, assetA, assetB) {
|
|
3317
|
-
|
|
3318
|
-
return { isValid: true, summary: 'No operations to validate' };
|
|
3319
|
-
}
|
|
3320
|
-
const { blockchainToFloat, floatToBlockchainInt, quantizeFloat } = require('./order/utils/math');
|
|
3321
|
-
const snap = this.manager.getChainFundsSnapshot();
|
|
3322
|
-
const netRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
|
|
3323
|
-
const runningRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
|
|
3324
|
-
const peakRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
|
|
3325
|
-
// Sum amounts and check individual order sizes
|
|
3326
|
-
for (const op of operations) {
|
|
3327
|
-
if (!op?.op_data)
|
|
3328
|
-
continue;
|
|
3329
|
-
let sellAssetId = null;
|
|
3330
|
-
let sellAmountInt = 0;
|
|
3331
|
-
if (op.op_name === 'limit_order_create') {
|
|
3332
|
-
sellAssetId = op.op_data.amount_to_sell?.asset_id;
|
|
3333
|
-
sellAmountInt = op.op_data.amount_to_sell?.amount;
|
|
3334
|
-
}
|
|
3335
|
-
else if (op.op_name === 'limit_order_update') {
|
|
3336
|
-
// In limit_order_update, new_price.base is the amount to sell
|
|
3337
|
-
sellAssetId = op.op_data.new_price?.base?.asset_id;
|
|
3338
|
-
sellAmountInt = op.op_data.new_price?.base?.amount;
|
|
3339
|
-
}
|
|
3340
|
-
if (sellAssetId && (sellAmountInt !== undefined && sellAmountInt !== null)) {
|
|
3341
|
-
const precision = (sellAssetId === assetA.id) ? assetA.precision : assetB.precision;
|
|
3342
|
-
const assetSymbol = (sellAssetId === assetA.id) ? assetA.symbol : assetB.symbol;
|
|
3343
|
-
// CRITICAL SAFETY CHECK: Ensure amount is greater than zero
|
|
3344
|
-
if (Number(sellAmountInt) <= 0) {
|
|
3345
|
-
return {
|
|
3346
|
-
isValid: false,
|
|
3347
|
-
summary: `[VALIDATION] CRITICAL: Zero amount order detected for ${assetSymbol} (assetId=${sellAssetId})`,
|
|
3348
|
-
violations: [{ asset: assetSymbol, sizeInt: sellAmountInt, reason: 'Zero amount' }]
|
|
3349
|
-
};
|
|
3350
|
-
}
|
|
3351
|
-
// Track signed per-op deltas in operation order.
|
|
3352
|
-
// CREATE consumes full amount; UPDATE consumes/releases delta.
|
|
3353
|
-
let signedDelta = 0;
|
|
3354
|
-
if (op.op_name === 'limit_order_update') {
|
|
3355
|
-
const deltaAssetId = op.op_data.delta_amount_to_sell?.asset_id;
|
|
3356
|
-
const deltaSellInt = op.op_data.delta_amount_to_sell?.amount;
|
|
3357
|
-
if (deltaAssetId === sellAssetId && Number.isFinite(Number(deltaSellInt))) {
|
|
3358
|
-
signedDelta = blockchainToFloat(deltaSellInt, precision);
|
|
3359
|
-
}
|
|
3360
|
-
}
|
|
3361
|
-
else {
|
|
3362
|
-
signedDelta = blockchainToFloat(sellAmountInt, precision);
|
|
3363
|
-
}
|
|
3364
|
-
netRequiredFunds[sellAssetId] = quantizeFloat((netRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
|
|
3365
|
-
runningRequiredFunds[sellAssetId] = quantizeFloat((runningRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
|
|
3366
|
-
const nextPeak = Math.max(Number(peakRequiredFunds[sellAssetId] || 0), Number(runningRequiredFunds[sellAssetId] || 0));
|
|
3367
|
-
peakRequiredFunds[sellAssetId] = quantizeFloat(nextPeak, precision);
|
|
3368
|
-
}
|
|
3369
|
-
}
|
|
3370
|
-
// Calculate available funds - CRITICAL FIX: Check against FREE balance, not free+required
|
|
3371
|
-
// Bug: Previous logic added requiredFunds to available, making validation meaningless
|
|
3372
|
-
// Correct logic: available = chainFree (current free balance)
|
|
3373
|
-
// If required > available, batch will fail on execution
|
|
3374
|
-
const availableFunds = {
|
|
3375
|
-
[assetA.id]: quantizeFloat(snap.chainFreeSell || 0, assetA.precision),
|
|
3376
|
-
[assetB.id]: quantizeFloat(snap.chainFreeBuy || 0, assetB.precision)
|
|
3377
|
-
};
|
|
3378
|
-
// Check for fund violations using quantized comparison
|
|
3379
|
-
const fundViolations = [];
|
|
3380
|
-
for (const assetId in peakRequiredFunds) {
|
|
3381
|
-
const required = peakRequiredFunds[assetId];
|
|
3382
|
-
const netRequired = netRequiredFunds[assetId] || 0;
|
|
3383
|
-
const available = availableFunds[assetId] || 0;
|
|
3384
|
-
// Use precision-aware comparison
|
|
3385
|
-
const prec = (assetId === assetA.id) ? assetA.precision : assetB.precision;
|
|
3386
|
-
if (floatToBlockchainInt(required, prec) > floatToBlockchainInt(available, prec)) {
|
|
3387
|
-
fundViolations.push({
|
|
3388
|
-
asset: assetId === assetA.id ? assetA.symbol : assetB.symbol,
|
|
3389
|
-
required,
|
|
3390
|
-
netRequired,
|
|
3391
|
-
available,
|
|
3392
|
-
deficit: quantizeFloat(required - available, prec)
|
|
3393
|
-
});
|
|
3394
|
-
}
|
|
3395
|
-
}
|
|
3396
|
-
if (fundViolations.length > 0) {
|
|
3397
|
-
let summary = `[VALIDATION] Fund validation FAILED:\n`;
|
|
3398
|
-
for (const v of fundViolations) {
|
|
3399
|
-
summary += ` ${v.asset}: peakRequired=${Format.formatAmount8(v.required)}, netRequired=${Format.formatAmount8(v.netRequired)}, available=${Format.formatAmount8(v.available)}, deficit=${Format.formatAmount8(v.deficit)}\n`;
|
|
3400
|
-
}
|
|
3401
|
-
return { isValid: false, summary: summary.trim(), violations: fundViolations };
|
|
3402
|
-
}
|
|
3403
|
-
const summary = `[VALIDATION] PASSED: ${operations.length} operations`;
|
|
3404
|
-
return { isValid: true, summary };
|
|
1007
|
+
return cowRuntime.validateOperationFunds(this, operations, assetA, assetB);
|
|
3405
1008
|
}
|
|
3406
1009
|
/**
|
|
3407
1010
|
* Resolve the ideal size from an order-like object with fallback.
|
|
@@ -3410,20 +1013,7 @@ class DEXBot {
|
|
|
3410
1013
|
* @returns {number|null} Resolved size or null
|
|
3411
1014
|
*/
|
|
3412
1015
|
_resolveIdealSizeForValidation(orderLike, fallbackSize = null) {
|
|
3413
|
-
|
|
3414
|
-
orderLike?.idealSize,
|
|
3415
|
-
orderLike?.order?.idealSize,
|
|
3416
|
-
orderLike?.size,
|
|
3417
|
-
orderLike?.order?.size,
|
|
3418
|
-
fallbackSize
|
|
3419
|
-
];
|
|
3420
|
-
for (const candidate of candidates) {
|
|
3421
|
-
const numeric = Number(candidate);
|
|
3422
|
-
if (Number.isFinite(numeric) && numeric > 0) {
|
|
3423
|
-
return numeric;
|
|
3424
|
-
}
|
|
3425
|
-
}
|
|
3426
|
-
return null;
|
|
1016
|
+
return cowRuntime.resolveIdealSizeForValidation(this, orderLike, fallbackSize);
|
|
3427
1017
|
}
|
|
3428
1018
|
/**
|
|
3429
1019
|
* Validate that an order size is safe to execute (above minimum dust thresholds).
|
|
@@ -3434,7 +1024,7 @@ class DEXBot {
|
|
|
3434
1024
|
* @returns {import('./types').OrderValidationResult}
|
|
3435
1025
|
*/
|
|
3436
1026
|
_validateOrderSizeForExecution(size, type, orderLike = null, fallbackSize = null) {
|
|
3437
|
-
return
|
|
1027
|
+
return cowRuntime.validateOrderSizeForExecution(this, size, type, orderLike, fallbackSize);
|
|
3438
1028
|
}
|
|
3439
1029
|
/**
|
|
3440
1030
|
* Execute a batch of order operations if the rebalance result has executable actions.
|
|
@@ -3774,77 +1364,7 @@ class DEXBot {
|
|
|
3774
1364
|
* @returns {Array<{type: string, id: string, order?: Object, orderId?: string, newSize?: number, newPrice?: number, newGridId?: string}>}
|
|
3775
1365
|
*/
|
|
3776
1366
|
_buildActionsFromPlan(plan) {
|
|
3777
|
-
|
|
3778
|
-
? { ordersToPlace: plan }
|
|
3779
|
-
: (plan || {});
|
|
3780
|
-
const { ordersToPlace = [], ordersToRotate = [], ordersToUpdate = [], ordersToCancel = [] } = normalizedPlan;
|
|
3781
|
-
const actions = [];
|
|
3782
|
-
for (const o of ordersToCancel) {
|
|
3783
|
-
if (o?.orderId) {
|
|
3784
|
-
actions.push({ type: COW_ACTIONS.CANCEL, id: o.id, orderId: o.orderId });
|
|
3785
|
-
}
|
|
3786
|
-
}
|
|
3787
|
-
for (const r of ordersToRotate) {
|
|
3788
|
-
const oldOrder = r?.oldOrder || r;
|
|
3789
|
-
const id = oldOrder?.id || r?.id;
|
|
3790
|
-
const orderId = oldOrder?.orderId || r?.orderId;
|
|
3791
|
-
const newGridId = r?.newGridId || id;
|
|
3792
|
-
const newSize = Number.isFinite(Number(r?.newSize))
|
|
3793
|
-
? Number(r.newSize)
|
|
3794
|
-
: Number(r?.size || oldOrder?.size || 0);
|
|
3795
|
-
const newPrice = Number.isFinite(Number(r?.newPrice))
|
|
3796
|
-
? Number(r.newPrice)
|
|
3797
|
-
: Number(r?.price || oldOrder?.price);
|
|
3798
|
-
const orderType = r?.type || oldOrder?.type;
|
|
3799
|
-
if (!id || !orderId || !newGridId || !orderType || !Number.isFinite(newPrice) || !(newSize > 0))
|
|
3800
|
-
continue;
|
|
3801
|
-
actions.push({
|
|
3802
|
-
type: COW_ACTIONS.UPDATE,
|
|
3803
|
-
id,
|
|
3804
|
-
orderId,
|
|
3805
|
-
newGridId,
|
|
3806
|
-
newSize,
|
|
3807
|
-
newPrice,
|
|
3808
|
-
order: {
|
|
3809
|
-
id: newGridId,
|
|
3810
|
-
type: orderType,
|
|
3811
|
-
price: newPrice,
|
|
3812
|
-
size: newSize
|
|
3813
|
-
}
|
|
3814
|
-
});
|
|
3815
|
-
}
|
|
3816
|
-
for (const o of ordersToUpdate) {
|
|
3817
|
-
const partialOrder = o?.partialOrder || o;
|
|
3818
|
-
const id = o?.id || partialOrder?.id;
|
|
3819
|
-
const orderId = o?.orderId || partialOrder?.orderId;
|
|
3820
|
-
const orderType = o?.type || partialOrder?.type;
|
|
3821
|
-
const newSize = Number.isFinite(Number(o?.newSize))
|
|
3822
|
-
? Number(o.newSize)
|
|
3823
|
-
: Number(partialOrder?.size || 0);
|
|
3824
|
-
if (!id || !orderId)
|
|
3825
|
-
continue;
|
|
3826
|
-
actions.push({
|
|
3827
|
-
type: COW_ACTIONS.UPDATE,
|
|
3828
|
-
id,
|
|
3829
|
-
orderId,
|
|
3830
|
-
newSize,
|
|
3831
|
-
order: {
|
|
3832
|
-
...(partialOrder || {}),
|
|
3833
|
-
id,
|
|
3834
|
-
orderId,
|
|
3835
|
-
type: orderType,
|
|
3836
|
-
size: newSize
|
|
3837
|
-
}
|
|
3838
|
-
});
|
|
3839
|
-
}
|
|
3840
|
-
// Run CREATE actions after UPDATE actions so same-batch downsizes can
|
|
3841
|
-
// release balance before placements consume it.
|
|
3842
|
-
for (const o of ordersToPlace) {
|
|
3843
|
-
if (!o?.id)
|
|
3844
|
-
continue;
|
|
3845
|
-
actions.push({ type: COW_ACTIONS.CREATE, id: o.id, order: o });
|
|
3846
|
-
}
|
|
3847
|
-
return actions;
|
|
1367
|
+
return cowRuntime.buildActionsFromPlan(this, plan);
|
|
3848
1368
|
}
|
|
3849
1369
|
/**
|
|
3850
1370
|
* Build a COW result object (workingGrid + actions) from a simple plan.
|
|
@@ -3852,78 +1372,7 @@ class DEXBot {
|
|
|
3852
1372
|
* @returns {{workingGrid: import('./types').WorkingGrid, workingIndexes: Object, workingBoundary: number, actions: Array}}
|
|
3853
1373
|
*/
|
|
3854
1374
|
_buildCowResultFromPlan(plan) {
|
|
3855
|
-
|
|
3856
|
-
const workingGrid = new WorkingGrid(this.manager.orders, {
|
|
3857
|
-
baseVersion: Number.isFinite(Number(this.manager._gridVersion)) ? this.manager._gridVersion : 0
|
|
3858
|
-
});
|
|
3859
|
-
const workingBoundary = this.manager.boundaryIdx;
|
|
3860
|
-
const actions = this._buildActionsFromPlan(plan);
|
|
3861
|
-
// Project planned actions into working grid so COW commit carries intended transitions.
|
|
3862
|
-
for (const action of actions) {
|
|
3863
|
-
if (action.type === COW_ACTIONS.CANCEL) {
|
|
3864
|
-
const current = workingGrid.get(action.id);
|
|
3865
|
-
if (!current)
|
|
3866
|
-
continue;
|
|
3867
|
-
workingGrid.set(action.id, convertToSpreadPlaceholder(current));
|
|
3868
|
-
}
|
|
3869
|
-
else if (action.type === COW_ACTIONS.CREATE) {
|
|
3870
|
-
if (!action.id || !action.order)
|
|
3871
|
-
continue;
|
|
3872
|
-
const current = workingGrid.get(action.id) || { id: action.id };
|
|
3873
|
-
workingGrid.set(action.id, {
|
|
3874
|
-
...current,
|
|
3875
|
-
...action.order,
|
|
3876
|
-
id: action.id,
|
|
3877
|
-
state: ORDER_STATES.VIRTUAL,
|
|
3878
|
-
orderId: null
|
|
3879
|
-
});
|
|
3880
|
-
}
|
|
3881
|
-
else if (action.type === COW_ACTIONS.UPDATE) {
|
|
3882
|
-
if (action.newGridId && action.newGridId !== action.id) {
|
|
3883
|
-
const current = workingGrid.get(action.id);
|
|
3884
|
-
if (current) {
|
|
3885
|
-
workingGrid.set(action.id, convertToSpreadPlaceholder(current));
|
|
3886
|
-
}
|
|
3887
|
-
const targetId = action.newGridId;
|
|
3888
|
-
const targetCurrent = workingGrid.get(targetId) || { id: targetId };
|
|
3889
|
-
const rotatedSize = Number.isFinite(Number(action.newSize))
|
|
3890
|
-
? Number(action.newSize)
|
|
3891
|
-
: Number(targetCurrent.size || 0);
|
|
3892
|
-
const rotatedPrice = Number.isFinite(Number(action.newPrice))
|
|
3893
|
-
? Number(action.newPrice)
|
|
3894
|
-
: Number(action.order?.price ?? targetCurrent.price);
|
|
3895
|
-
workingGrid.set(targetId, {
|
|
3896
|
-
...targetCurrent,
|
|
3897
|
-
...(action.order || {}),
|
|
3898
|
-
id: targetId,
|
|
3899
|
-
size: rotatedSize,
|
|
3900
|
-
price: rotatedPrice,
|
|
3901
|
-
state: ORDER_STATES.VIRTUAL,
|
|
3902
|
-
orderId: null
|
|
3903
|
-
});
|
|
3904
|
-
continue;
|
|
3905
|
-
}
|
|
3906
|
-
const current = workingGrid.get(action.id);
|
|
3907
|
-
if (!current)
|
|
3908
|
-
continue;
|
|
3909
|
-
const newSize = Number.isFinite(Number(action.newSize))
|
|
3910
|
-
? Number(action.newSize)
|
|
3911
|
-
: Number(current.size || 0);
|
|
3912
|
-
workingGrid.set(action.id, {
|
|
3913
|
-
...current,
|
|
3914
|
-
...(action.order || {}),
|
|
3915
|
-
id: action.id,
|
|
3916
|
-
orderId: action.orderId || current.orderId,
|
|
3917
|
-
size: newSize
|
|
3918
|
-
});
|
|
3919
|
-
}
|
|
3920
|
-
}
|
|
3921
|
-
return {
|
|
3922
|
-
workingGrid,
|
|
3923
|
-
workingIndexes: workingGrid.getIndexes(),
|
|
3924
|
-
workingBoundary,
|
|
3925
|
-
actions
|
|
3926
|
-
};
|
|
1375
|
+
return cowRuntime.buildCowResultFromPlan(this, plan);
|
|
3927
1376
|
}
|
|
3928
1377
|
/**
|
|
3929
1378
|
* Restore skipped update slots in the working grid to master state.
|
|
@@ -3933,16 +1382,7 @@ class DEXBot {
|
|
|
3933
1382
|
* @returns {void}
|
|
3934
1383
|
*/
|
|
3935
1384
|
_restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedSlotIds, skippedCount = 0) {
|
|
3936
|
-
|
|
3937
|
-
return;
|
|
3938
|
-
}
|
|
3939
|
-
const masterVersion = Number.isFinite(Number(this.manager?._gridVersion))
|
|
3940
|
-
? Number(this.manager._gridVersion)
|
|
3941
|
-
: undefined;
|
|
3942
|
-
for (const slotId of skippedSlotIds) {
|
|
3943
|
-
workingGrid.syncFromMaster(this.manager.orders, slotId, masterVersion);
|
|
3944
|
-
}
|
|
3945
|
-
this.manager.logger.log(`[COW] Restored ${skippedSlotIds.size} slot(s) after ${skippedCount} skipped update action(s).`, 'debug');
|
|
1385
|
+
return cowRuntime.restoreSkippedUpdateSlotsInWorkingGrid(this, workingGrid, skippedSlotIds, skippedCount);
|
|
3946
1386
|
}
|
|
3947
1387
|
/**
|
|
3948
1388
|
* COW broadcast: Execute blockchain operations and commit working grid on success.
|
|
@@ -3952,751 +1392,10 @@ class DEXBot {
|
|
|
3952
1392
|
* @private
|
|
3953
1393
|
*/
|
|
3954
1394
|
async _updateOrdersOnChainBatchCOW(cowResult) {
|
|
3955
|
-
|
|
3956
|
-
const { workingGrid, workingIndexes, workingBoundary, actions } = cowResult;
|
|
3957
|
-
if (this.config.dryRun) {
|
|
3958
|
-
const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
|
|
3959
|
-
const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
|
|
3960
|
-
const updateCount = actions.filter(a => a.type === COW_ACTIONS.UPDATE).length;
|
|
3961
|
-
if (cancelCount > 0)
|
|
3962
|
-
this.manager.logger.log(`Dry run: would cancel ${cancelCount} orders`, 'info');
|
|
3963
|
-
if (createCount > 0)
|
|
3964
|
-
this.manager.logger.log(`Dry run: would place ${createCount} new orders`, 'info');
|
|
3965
|
-
if (updateCount > 0)
|
|
3966
|
-
this.manager.logger.log(`Dry run: would update ${updateCount} orders`, 'info');
|
|
3967
|
-
return { executed: true, hadRotation: false };
|
|
3968
|
-
}
|
|
3969
|
-
const createSlotValidation = validateCreateTargetSlots(actions, this.manager?.orders);
|
|
3970
|
-
if (!createSlotValidation.isValid) {
|
|
3971
|
-
for (const violation of createSlotValidation.violations) {
|
|
3972
|
-
this.manager.logger.log(`[COW] Rejecting CREATE for occupied slot ${violation.targetId}: ` +
|
|
3973
|
-
`existing orderId=${violation.currentOrderId}, type=${violation.currentType}, state=${violation.currentState}`, 'error');
|
|
3974
|
-
}
|
|
3975
|
-
return {
|
|
3976
|
-
executed: false,
|
|
3977
|
-
aborted: true,
|
|
3978
|
-
reason: 'CREATE_SLOT_OCCUPIED',
|
|
3979
|
-
violations: createSlotValidation.violations,
|
|
3980
|
-
hadRotation: false
|
|
3981
|
-
};
|
|
3982
|
-
}
|
|
3983
|
-
const hasCreateActions = actions.some(action => action.type === COW_ACTIONS.CREATE);
|
|
3984
|
-
// Gap 5: Recovery exhausted — block all CREATES. The bot has exhausted
|
|
3985
|
-
// its recovery attempt budget and needs a fill or sync cycle to reset.
|
|
3986
|
-
// Existing orders remain active and are monitored, but no new orders
|
|
3987
|
-
// are placed until recovery resets.
|
|
3988
|
-
if (hasCreateActions && this.manager?._recoveryExhaustedAt) {
|
|
3989
|
-
const exhaustedAge = Date.now() - this.manager._recoveryExhaustedAt;
|
|
3990
|
-
this.manager.logger.log?.(`[RECOVERY-EXHAUSTED] Blocking ${actions.filter(a => a.type === COW_ACTIONS.CREATE).length} CREATE(s) ` +
|
|
3991
|
-
`(exhausted ${(exhaustedAge / 1000).toFixed(0)}s ago). ` +
|
|
3992
|
-
`Waiting for next fill or sync cycle to reset recovery state.`, 'warn');
|
|
3993
|
-
return {
|
|
3994
|
-
executed: false,
|
|
3995
|
-
aborted: true,
|
|
3996
|
-
reason: 'RECOVERY_EXHAUSTED',
|
|
3997
|
-
hadRotation: false
|
|
3998
|
-
};
|
|
3999
|
-
}
|
|
4000
|
-
const unmatchedChainOrders = Array.isArray(this.manager?._lastUnmatchedChainOrders)
|
|
4001
|
-
? this.manager._lastUnmatchedChainOrders
|
|
4002
|
-
: [];
|
|
4003
|
-
// PENDING_BROADCASTS: an earlier batch timed out at the credential
|
|
4004
|
-
// daemon and the chain status of its CREATEs is still unknown. The
|
|
4005
|
-
// recovery path (_reconcileAfterUncertainBroadcast) is responsible
|
|
4006
|
-
// for resolving it. We refuse to publish a fresh CREATE batch until
|
|
4007
|
-
// recovery runs, otherwise we risk double-publishing or stacking
|
|
4008
|
-
// orphan orders on top of potentially-orphaned ones.
|
|
4009
|
-
const pendingBroadcasts = (this.manager && this.manager._pendingBroadcasts instanceof Map)
|
|
4010
|
-
? Array.from(this.manager._pendingBroadcasts.values())
|
|
4011
|
-
: [];
|
|
4012
|
-
if (hasCreateActions && (unmatchedChainOrders.length > 0 || pendingBroadcasts.length > 0)) {
|
|
4013
|
-
// ---- Handle pending broadcasts first ----
|
|
4014
|
-
if (pendingBroadcasts.length > 0) {
|
|
4015
|
-
this.manager.logger.log(`[COW] Rejecting CREATE batch: ${pendingBroadcasts.length} pending broadcast(s) from a prior uncertain ` +
|
|
4016
|
-
`broadcast. Running recovery before placing replacement orders.`, 'error');
|
|
4017
|
-
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
4018
|
-
if (this.manager._recoveryState)
|
|
4019
|
-
this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
|
|
4020
|
-
await this.manager.requestStructuralGridResync('pending broadcasts before COW create', { pendingBroadcasts: pendingBroadcasts.map(p => p.slotId) });
|
|
4021
|
-
}
|
|
4022
|
-
try {
|
|
4023
|
-
await this._reconcileAfterUncertainBroadcast(new BroadcastUncertainError('rejected CREATE batch had pending broadcasts', {
|
|
4024
|
-
operations: pendingBroadcasts.map(p => p.order),
|
|
4025
|
-
accountName: this.account,
|
|
4026
|
-
batchId: this._currentBatchId || null,
|
|
4027
|
-
payload: null,
|
|
4028
|
-
timeoutMs: null
|
|
4029
|
-
}), []);
|
|
4030
|
-
}
|
|
4031
|
-
catch (recoverErr) {
|
|
4032
|
-
this.manager.logger.log(`[COW] Recovery from pending broadcasts failed: ${recoverErr?.message || recoverErr}`, 'error');
|
|
4033
|
-
}
|
|
4034
|
-
return {
|
|
4035
|
-
executed: false,
|
|
4036
|
-
aborted: true,
|
|
4037
|
-
reason: 'PENDING_BROADCASTS',
|
|
4038
|
-
hadRotation: false
|
|
4039
|
-
};
|
|
4040
|
-
}
|
|
4041
|
-
// ---- Unmatched orders: adopt instead of cancel ----
|
|
4042
|
-
// Unmatched orders are legitimate on-chain positions the grid
|
|
4043
|
-
// hasn't adopted yet. Cancelling them destroys capital and
|
|
4044
|
-
// creates gaps in the order book. Instead, re-sync to adopt
|
|
4045
|
-
// them into the grid, then reject (the sync invalidated the
|
|
4046
|
-
// working grid, so the existing COW plan is stale).
|
|
4047
|
-
// NOTE: syncFromOpenOrders may be a partial no-op under certain
|
|
4048
|
-
// lock conditions (e.g. _fillProcessingLock + !isReentrant).
|
|
4049
|
-
// Structural resync scheduled below handles adoption regardless.
|
|
4050
|
-
const unmatchedSample = unmatchedChainOrders
|
|
4051
|
-
.slice(0, 3)
|
|
4052
|
-
.map(o => this._formatUnmatchedChainOrderForLog(o))
|
|
4053
|
-
.join(' | ');
|
|
4054
|
-
this.manager.logger.log(`[COW] ${unmatchedChainOrders.length} unmatched chain order(s) blocking CREATES ` +
|
|
4055
|
-
(unmatchedSample ? `(${unmatchedSample})` : '') +
|
|
4056
|
-
` — adopting via sync instead of cancelling`, 'info');
|
|
4057
|
-
try {
|
|
4058
|
-
const accountRef = this.account;
|
|
4059
|
-
const freshSnapshot = await chainOrders.readOpenOrders(accountRef);
|
|
4060
|
-
if (freshSnapshot && freshSnapshot.length > 0) {
|
|
4061
|
-
const syncResult = await this.manager.syncFromOpenOrders(freshSnapshot, {
|
|
4062
|
-
skipAccounting: true,
|
|
4063
|
-
});
|
|
4064
|
-
// Only overwrite _lastUnmatchedChainOrders when sync actually
|
|
4065
|
-
// processed orders (filled + updated + corrected > 0).
|
|
4066
|
-
// Force-release and lock-contention early-exit paths return
|
|
4067
|
-
// empty arrays without touching _lastUnmatchedChainOrders —
|
|
4068
|
-
// overwriting with [] would drop stale unmatched entries.
|
|
4069
|
-
if (syncResult && Array.isArray(syncResult.unmatchedChainOrders)) {
|
|
4070
|
-
const processed = (syncResult.filledOrders?.length || 0) +
|
|
4071
|
-
(syncResult.updatedOrders?.length || 0) +
|
|
4072
|
-
(syncResult.ordersNeedingCorrection?.length || 0);
|
|
4073
|
-
if (processed > 0) {
|
|
4074
|
-
this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders;
|
|
4075
|
-
this.manager.logger.log(`[COW] Adopted chain order(s) via sync: ${processed} processed, ` +
|
|
4076
|
-
`${syncResult.unmatchedChainOrders.length} still unmatched`, 'info');
|
|
4077
|
-
}
|
|
4078
|
-
else {
|
|
4079
|
-
// processed === 0 with unmatched still present means the
|
|
4080
|
-
// sync could not adopt the unmatched chain orders. This is
|
|
4081
|
-
// normal when called re-entrantly (inside _fillProcessingLock):
|
|
4082
|
-
// the sync engine runs inline and either timed out (unlikely
|
|
4083
|
-
// for a re-entrant call) or all chain orders were already
|
|
4084
|
-
// matched — leaving only stale unmatched entries.
|
|
4085
|
-
const syncUnmatchedCount = syncResult.unmatchedChainOrders.length;
|
|
4086
|
-
this.manager.logger.log(`[COW] Sync returned without processing (processed=0, ` +
|
|
4087
|
-
`unmatched=${syncUnmatchedCount} in result, ` +
|
|
4088
|
-
`_lastUnmatchedChainOrders=${unmatchedChainOrders.length}). ` +
|
|
4089
|
-
`Structural resync will handle adoption.`, syncUnmatchedCount > 0 ? 'warn' : 'debug');
|
|
4090
|
-
// If the sync result itself has unmatched entries that
|
|
4091
|
-
// differ from _lastUnmatchedChainOrders, adopt them now
|
|
4092
|
-
// so the stale tracker is more accurate for the resync.
|
|
4093
|
-
if (syncUnmatchedCount > 0 && syncUnmatchedCount !== unmatchedChainOrders.length) {
|
|
4094
|
-
this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders.map((o) => ({ ...o }));
|
|
4095
|
-
this.manager.logger.log(`[COW] Updated _lastUnmatchedChainOrders from sync result: ` +
|
|
4096
|
-
`${unmatchedChainOrders.length} → ${syncUnmatchedCount}`, 'debug');
|
|
4097
|
-
}
|
|
4098
|
-
}
|
|
4099
|
-
}
|
|
4100
|
-
}
|
|
4101
|
-
}
|
|
4102
|
-
catch (syncErr) {
|
|
4103
|
-
this.manager.logger.log(`[COW] Failed to sync/unmatched orders: ${syncErr?.message || syncErr}`, 'warn');
|
|
4104
|
-
}
|
|
4105
|
-
// Working grid is stale after master grid mutation from sync.
|
|
4106
|
-
// Request structural resync to rebuild the grid on the next cycle.
|
|
4107
|
-
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
4108
|
-
if (this.manager._recoveryState)
|
|
4109
|
-
this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
|
|
4110
|
-
await this.manager.requestStructuralGridResync('unmatched chain orders before COW create', { unmatchedChainOrders: unmatchedChainOrders });
|
|
4111
|
-
}
|
|
4112
|
-
this.manager.logger.log(`[COW] Rejecting CREATE batch after sync: working grid invalidated by master mutation`, 'info');
|
|
4113
|
-
return {
|
|
4114
|
-
executed: false,
|
|
4115
|
-
aborted: true,
|
|
4116
|
-
reason: 'UNMATCHED_CHAIN_ORDERS',
|
|
4117
|
-
hadRotation: false
|
|
4118
|
-
};
|
|
4119
|
-
}
|
|
4120
|
-
const { assetA, assetB } = this.manager.assets;
|
|
4121
|
-
const operations = [];
|
|
4122
|
-
const opContexts = [];
|
|
4123
|
-
const skippedUpdateSlotIds = new Set();
|
|
4124
|
-
let skippedUpdateCount = 0;
|
|
4125
|
-
// Collect IDs to lock from actions
|
|
4126
|
-
const idsToLock = new Set();
|
|
4127
|
-
for (const action of actions) {
|
|
4128
|
-
if (action.type === COW_ACTIONS.CANCEL && action.orderId) {
|
|
4129
|
-
idsToLock.add(action.orderId);
|
|
4130
|
-
if (action.id)
|
|
4131
|
-
idsToLock.add(action.id);
|
|
4132
|
-
}
|
|
4133
|
-
else if (action.type === COW_ACTIONS.CREATE && action.id) {
|
|
4134
|
-
idsToLock.add(action.id);
|
|
4135
|
-
}
|
|
4136
|
-
else if (action.type === COW_ACTIONS.UPDATE && action.orderId) {
|
|
4137
|
-
idsToLock.add(action.orderId);
|
|
4138
|
-
if (action.id)
|
|
4139
|
-
idsToLock.add(action.id);
|
|
4140
|
-
}
|
|
4141
|
-
}
|
|
4142
|
-
// Apply shadow locks
|
|
4143
|
-
this.manager.lockOrders(idsToLock);
|
|
4144
|
-
try {
|
|
4145
|
-
this._batchInFlight = true;
|
|
4146
|
-
this._markGridActivity('batch start');
|
|
4147
|
-
this.manager._setRebalanceState(REBALANCE_STATES.BROADCASTING);
|
|
4148
|
-
this.manager.startBroadcasting();
|
|
4149
|
-
// Build operations from actions
|
|
4150
|
-
for (const action of actions) {
|
|
4151
|
-
if (action.type === COW_ACTIONS.CANCEL) {
|
|
4152
|
-
try {
|
|
4153
|
-
const op = await chainOrders.buildCancelOrderOp(this.account, action.orderId);
|
|
4154
|
-
operations.push(op);
|
|
4155
|
-
const order = this.manager.orders.get(action.id) || { id: action.id, orderId: action.orderId };
|
|
4156
|
-
opContexts.push({ kind: 'cancel', order });
|
|
4157
|
-
}
|
|
4158
|
-
catch (err) {
|
|
4159
|
-
const orderNotFound = /\bnot found\b/i.test(err.message) || /\bdoes not exist\b/i.test(err.message);
|
|
4160
|
-
if (orderNotFound) {
|
|
4161
|
-
this.manager.logger.log(`[COW] Cancel skipped for ${action.id} (${action.orderId}): order already removed from chain`, 'debug');
|
|
4162
|
-
}
|
|
4163
|
-
else {
|
|
4164
|
-
this.manager.logger.log(`Failed to prepare cancel op for ${action.id}: ${err.message}`, 'error');
|
|
4165
|
-
}
|
|
4166
|
-
}
|
|
4167
|
-
}
|
|
4168
|
-
else if (action.type === COW_ACTIONS.CREATE) {
|
|
4169
|
-
try {
|
|
4170
|
-
const order = action.order;
|
|
4171
|
-
const sizeValidation = this._validateOrderSizeForExecution(order.size, order.type, order, order.size);
|
|
4172
|
-
if (!sizeValidation.isValid) {
|
|
4173
|
-
this.manager.logger.log(`Skipping create op for ${action.id}: ${sizeValidation.reason}`, 'warn');
|
|
4174
|
-
continue;
|
|
4175
|
-
}
|
|
4176
|
-
const liveSlot = this.manager.orders.get(order.id);
|
|
4177
|
-
const plannedPrice = Number(order.price);
|
|
4178
|
-
const livePrice = liveSlot ? Number(liveSlot.price) : NaN;
|
|
4179
|
-
const priceDrift = Number.isFinite(plannedPrice) && Number.isFinite(livePrice)
|
|
4180
|
-
? Math.abs(livePrice - plannedPrice)
|
|
4181
|
-
: 0;
|
|
4182
|
-
const effectiveOrder = (priceDrift > 0)
|
|
4183
|
-
? { ...order, price: livePrice, size: order.size, type: order.type }
|
|
4184
|
-
: order;
|
|
4185
|
-
if (priceDrift > 0) {
|
|
4186
|
-
this.manager.logger.log(`[COW] Pre-broadcast price freshness: slot ${order.id} ` +
|
|
4187
|
-
`drifted from planned=${plannedPrice} to live=${livePrice} ` +
|
|
4188
|
-
`(diff=${priceDrift}); rebuilding CREATE op with live price.`, 'debug');
|
|
4189
|
-
}
|
|
4190
|
-
const args = buildCreateOrderArgs(effectiveOrder, assetA, assetB);
|
|
4191
|
-
const buildResult = await chainOrders.buildCreateOrderOp(this.account, args.amountToSell, args.sellAssetId, args.minToReceive, args.receiveAssetId, null);
|
|
4192
|
-
if (!buildResult) {
|
|
4193
|
-
this.manager.logger.log(`Skipping create op for ${action.id}: amounts would round to 0 on blockchain`, 'warn');
|
|
4194
|
-
continue;
|
|
4195
|
-
}
|
|
4196
|
-
operations.push(buildResult.op);
|
|
4197
|
-
opContexts.push({ kind: 'create', id: order.id, order: effectiveOrder, args, finalInts: buildResult.finalInts });
|
|
4198
|
-
this._recordPendingBroadcast({
|
|
4199
|
-
opIndex: operations.length - 1,
|
|
4200
|
-
ctxIndex: opContexts.length - 1,
|
|
4201
|
-
order: effectiveOrder,
|
|
4202
|
-
finalInts: buildResult.finalInts
|
|
4203
|
-
});
|
|
4204
|
-
}
|
|
4205
|
-
catch (err) {
|
|
4206
|
-
this.manager.logger.log(`Failed to prepare create op for ${action.id}: ${err.message}`, 'error');
|
|
4207
|
-
}
|
|
4208
|
-
}
|
|
4209
|
-
else if (action.type === COW_ACTIONS.UPDATE) {
|
|
4210
|
-
try {
|
|
4211
|
-
// Rotation update: move existing on-chain order to a new slot/price.
|
|
4212
|
-
if (action.newGridId && action.newGridId !== action.id) {
|
|
4213
|
-
const masterOrder = this.manager.orders.get(action.id);
|
|
4214
|
-
const orderType = action.order?.type || masterOrder?.type;
|
|
4215
|
-
const newPrice = Number.isFinite(Number(action.newPrice))
|
|
4216
|
-
? Number(action.newPrice)
|
|
4217
|
-
: Number(action.order?.price);
|
|
4218
|
-
const newSize = Number.isFinite(Number(action.newSize))
|
|
4219
|
-
? Number(action.newSize)
|
|
4220
|
-
: Number(action.order?.size || 0);
|
|
4221
|
-
if (!masterOrder || !action.orderId || !orderType || !Number.isFinite(newPrice) || newSize <= 0) {
|
|
4222
|
-
continue;
|
|
4223
|
-
}
|
|
4224
|
-
const rotationSizeValidation = this._validateOrderSizeForExecution(newSize, orderType, action.order, newSize);
|
|
4225
|
-
if (!rotationSizeValidation.isValid) {
|
|
4226
|
-
this.manager.logger.log(`Skipping rotation update ${action.id} -> ${action.newGridId}: ${rotationSizeValidation.reason}`, 'warn');
|
|
4227
|
-
continue;
|
|
4228
|
-
}
|
|
4229
|
-
const { amountToSell, minToReceive } = buildCreateOrderArgs({ type: orderType, size: newSize, price: newPrice }, assetA, assetB);
|
|
4230
|
-
const buildResult = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell, minToReceive, newPrice, orderType }, masterOrder.rawOnChain || null);
|
|
4231
|
-
if (!buildResult) {
|
|
4232
|
-
skippedUpdateCount++;
|
|
4233
|
-
if (action.id)
|
|
4234
|
-
skippedUpdateSlotIds.add(action.id);
|
|
4235
|
-
if (action.newGridId)
|
|
4236
|
-
skippedUpdateSlotIds.add(action.newGridId);
|
|
4237
|
-
this.manager.logger.log(`[COW] Skipping rotation update ${action.id} -> ${action.newGridId}: no blockchain delta`, 'debug');
|
|
4238
|
-
continue;
|
|
4239
|
-
}
|
|
4240
|
-
operations.push(buildResult.op);
|
|
4241
|
-
opContexts.push({
|
|
4242
|
-
kind: 'rotation',
|
|
4243
|
-
rotation: {
|
|
4244
|
-
oldOrder: { ...masterOrder },
|
|
4245
|
-
newGridId: action.newGridId,
|
|
4246
|
-
newPrice,
|
|
4247
|
-
newSize,
|
|
4248
|
-
type: orderType
|
|
4249
|
-
},
|
|
4250
|
-
finalInts: buildResult.finalInts
|
|
4251
|
-
});
|
|
4252
|
-
continue;
|
|
4253
|
-
}
|
|
4254
|
-
const newSize = Number.isFinite(Number(action.newSize))
|
|
4255
|
-
? Number(action.newSize)
|
|
4256
|
-
: Number(action.order?.size || 0);
|
|
4257
|
-
const masterOrder = this.manager.orders.get(action.id);
|
|
4258
|
-
const orderType = action.order?.type || masterOrder?.type;
|
|
4259
|
-
const cachedRawOnChain = masterOrder?.rawOnChain || action.order?.rawOnChain || null;
|
|
4260
|
-
const op = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell: newSize, orderType }, cachedRawOnChain);
|
|
4261
|
-
if (!op) {
|
|
4262
|
-
skippedUpdateCount++;
|
|
4263
|
-
if (action.id)
|
|
4264
|
-
skippedUpdateSlotIds.add(action.id);
|
|
4265
|
-
if (action.newGridId)
|
|
4266
|
-
skippedUpdateSlotIds.add(action.newGridId);
|
|
4267
|
-
this.manager.logger.log(`[COW] Skipping size update ${action.id} (${action.orderId}): no blockchain delta`, 'debug');
|
|
4268
|
-
continue;
|
|
4269
|
-
}
|
|
4270
|
-
operations.push(op.op);
|
|
4271
|
-
const partialOrder = masterOrder || {
|
|
4272
|
-
id: action.id,
|
|
4273
|
-
orderId: action.orderId,
|
|
4274
|
-
type: orderType
|
|
4275
|
-
};
|
|
4276
|
-
opContexts.push({ kind: 'size-update', updateInfo: { partialOrder, newSize }, finalInts: op.finalInts });
|
|
4277
|
-
// Catch for both rotation-update and size-update branches.
|
|
4278
|
-
}
|
|
4279
|
-
catch (err) {
|
|
4280
|
-
const orderNotFound = /\bnot found\b/i.test(err.message) || /\bdoes not exist\b/i.test(err.message);
|
|
4281
|
-
if (orderNotFound) {
|
|
4282
|
-
try {
|
|
4283
|
-
const fbOrder = action.order || this.manager.orders.get(action.id);
|
|
4284
|
-
const fbType = fbOrder?.type;
|
|
4285
|
-
const fbSize = action.newSize || fbOrder?.size || 0;
|
|
4286
|
-
// Live price from TARGET slot (same pattern as CREATE path).
|
|
4287
|
-
const targetSlotId = action.newGridId || action.id;
|
|
4288
|
-
const plannedPrice = action.newPrice || action.order?.price || 0;
|
|
4289
|
-
const liveSlotForPrice = this.manager.orders.get(targetSlotId);
|
|
4290
|
-
const livePrice = liveSlotForPrice ? Number(liveSlotForPrice.price) : NaN;
|
|
4291
|
-
const priceDrift = Number.isFinite(plannedPrice) && Number.isFinite(livePrice)
|
|
4292
|
-
? Math.abs(livePrice - plannedPrice)
|
|
4293
|
-
: 0;
|
|
4294
|
-
const fbPrice = (priceDrift > 0) ? livePrice : plannedPrice;
|
|
4295
|
-
if (priceDrift > 0) {
|
|
4296
|
-
this.manager.logger.log(`[COW] CREATE fallback price drift for ${action.id} -> ${targetSlotId}: ` +
|
|
4297
|
-
`planned=${plannedPrice} live=${livePrice} (diff=${priceDrift})`, 'debug');
|
|
4298
|
-
}
|
|
4299
|
-
// Same size gate as regular CREATE path.
|
|
4300
|
-
const sizeCheck = this._validateOrderSizeForExecution(fbSize, fbType, fbOrder, fbSize);
|
|
4301
|
-
if (!sizeCheck.isValid) {
|
|
4302
|
-
this.manager.logger.log(`[COW] CREATE fallback for ${action.id} rejected by size validation: ${sizeCheck.reason}`, 'warn');
|
|
4303
|
-
}
|
|
4304
|
-
else if (fbType && fbSize > 0 && fbPrice > 0) {
|
|
4305
|
-
const fbArgs = buildCreateOrderArgs({ type: fbType, size: fbSize, price: fbPrice }, assetA, assetB);
|
|
4306
|
-
const fbResult = await chainOrders.buildCreateOrderOp(this.account, fbArgs.amountToSell, fbArgs.sellAssetId, fbArgs.minToReceive, fbArgs.receiveAssetId, null);
|
|
4307
|
-
if (fbResult) {
|
|
4308
|
-
operations.push(fbResult.op);
|
|
4309
|
-
opContexts.push({
|
|
4310
|
-
kind: 'create',
|
|
4311
|
-
id: targetSlotId,
|
|
4312
|
-
order: { id: targetSlotId, type: fbType, price: fbPrice, size: fbSize },
|
|
4313
|
-
args: { amountToSell: fbArgs.amountToSell, minToReceive: fbArgs.minToReceive },
|
|
4314
|
-
finalInts: fbResult.finalInts
|
|
4315
|
-
});
|
|
4316
|
-
this._recordPendingBroadcast({
|
|
4317
|
-
opIndex: operations.length - 1,
|
|
4318
|
-
ctxIndex: opContexts.length - 1,
|
|
4319
|
-
order: { id: targetSlotId, type: fbType, price: fbPrice, size: fbSize },
|
|
4320
|
-
finalInts: fbResult.finalInts
|
|
4321
|
-
});
|
|
4322
|
-
this.manager.logger.log(`[COW] Recovered "not found" for ${action.id}: converted UPDATE to CREATE for slot ${targetSlotId}`, 'warn');
|
|
4323
|
-
continue;
|
|
4324
|
-
}
|
|
4325
|
-
}
|
|
4326
|
-
}
|
|
4327
|
-
catch (fbErr) {
|
|
4328
|
-
this.manager.logger.log(`[COW] CREATE fallback also failed for ${action.id}: ${fbErr.message}`, 'warn');
|
|
4329
|
-
}
|
|
4330
|
-
}
|
|
4331
|
-
this.manager.logger.log(`Failed to prepare update op for ${action.id}: ${err.message}`, 'error');
|
|
4332
|
-
}
|
|
4333
|
-
}
|
|
4334
|
-
}
|
|
4335
|
-
if (skippedUpdateCount > 0) {
|
|
4336
|
-
this._restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedUpdateSlotIds, skippedUpdateCount);
|
|
4337
|
-
}
|
|
4338
|
-
if (operations.length === 0) {
|
|
4339
|
-
this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
|
|
4340
|
-
return { executed: false, hadRotation: false };
|
|
4341
|
-
}
|
|
4342
|
-
// Validate funds before broadcasting
|
|
4343
|
-
const validation = this._validateOperationFunds(operations, assetA, assetB);
|
|
4344
|
-
this.manager.logger.log(validation.summary, validation.isValid ? 'info' : 'warn');
|
|
4345
|
-
if (!validation.isValid) {
|
|
4346
|
-
this.manager.logger.log(`Skipping batch broadcast: ${validation.violations.length} fund violation(s) detected`, 'warn');
|
|
4347
|
-
this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
|
|
4348
|
-
return { executed: false, hadRotation: false };
|
|
4349
|
-
}
|
|
4350
|
-
await this._ensureCredentialDaemonWritable('COW batch broadcast');
|
|
4351
|
-
// Execute batch
|
|
4352
|
-
this.manager.logger.log(`[COW] Broadcasting batch with ${operations.length} operations...`, 'info');
|
|
4353
|
-
this._lastBroadcastHeartbeatAt = Date.now();
|
|
4354
|
-
const execution = await this._executeWithRetryOnUncertain(operations, opContexts);
|
|
4355
|
-
const result = execution.result;
|
|
4356
|
-
const executedContexts = execution.opContexts;
|
|
4357
|
-
// Process results and commit on success
|
|
4358
|
-
this.manager.pauseFundRecalc();
|
|
4359
|
-
try {
|
|
4360
|
-
this.manager._throwOnIllegalState = true;
|
|
4361
|
-
if (result.success) {
|
|
4362
|
-
// Pre-commit integrity: only CREATE ops require returned chainOrderIds.
|
|
4363
|
-
// Cancel/update operation results may be empty depending on the broadcaster.
|
|
4364
|
-
const preCommitResults = this._extractOperationResults(result, 'pre-commit-integrity');
|
|
4365
|
-
const missingCreateResults = this._findMissingCreateResultContexts(preCommitResults, executedContexts);
|
|
4366
|
-
if (missingCreateResults.length > 0) {
|
|
4367
|
-
const missingSlots = missingCreateResults
|
|
4368
|
-
.map(item => item.ctx?.order?.id || item.ctx?.id || `op-${item.index}`)
|
|
4369
|
-
.join(', ');
|
|
4370
|
-
this.manager.logger.log(`[COW] Refusing to commit working grid: ${missingCreateResults.length} CREATE op(s) ` +
|
|
4371
|
-
`returned no chainOrderId (${missingSlots}). Discarding working grid and syncing from chain.`, 'error');
|
|
4372
|
-
this.manager._clearWorkingGridRef();
|
|
4373
|
-
this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
|
|
4374
|
-
this._markMissingCreateResultsAsStructuralBlocker(missingCreateResults);
|
|
4375
|
-
await this._recoverAfterMissingCreateResults('missing create operation results');
|
|
4376
|
-
return {
|
|
4377
|
-
executed: false,
|
|
4378
|
-
hadRotation: false,
|
|
4379
|
-
missingCreateResults: missingCreateResults.map(item => ({
|
|
4380
|
-
index: item.index,
|
|
4381
|
-
slotId: item.ctx?.order?.id || item.ctx?.id || null
|
|
4382
|
-
}))
|
|
4383
|
-
};
|
|
4384
|
-
}
|
|
4385
|
-
// SUCCESS: Commit working grid to master (atomic swap)
|
|
4386
|
-
this.manager.logger.log('[COW] Blockchain success - committing working grid to master', 'info');
|
|
4387
|
-
// RC-FIX: skipRecalc prevents invariant violation before optimistic accounting
|
|
4388
|
-
await this.manager._commitWorkingGrid(workingGrid, workingIndexes, workingBoundary, { skipRecalc: true });
|
|
4389
|
-
// Commitment accounting is handled in real-time by
|
|
4390
|
-
// updateOptimisticFreeBalance when capital is committed to orders.
|
|
4391
|
-
// The old post-batch deduction path was removed to avoid double-counting.
|
|
4392
|
-
// Process batch results for logging/metrics
|
|
4393
|
-
const batchResult = await this._processBatchResults(result, executedContexts);
|
|
4394
|
-
// Persist to disk. CRITICAL: working grid was already committed
|
|
4395
|
-
// to master above; if persistence is skipped or validation fails
|
|
4396
|
-
// here, the on-disk snapshot will be older than the in-memory
|
|
4397
|
-
// master. Retry once and surface the failure explicitly so the
|
|
4398
|
-
// next cycle / shutdown can recover.
|
|
4399
|
-
const persistResult = await this.manager.persistGrid();
|
|
4400
|
-
if (persistResult && (persistResult.skipped || persistResult.isValid === false)) {
|
|
4401
|
-
this.manager.logger.log(`[COW][PERSIST-GUARD] First persist attempt was ` +
|
|
4402
|
-
`${persistResult.skipped ? 'skipped' : 'invalid'} ` +
|
|
4403
|
-
`(${persistResult.reason || 'no reason'}); retrying once before ` +
|
|
4404
|
-
`clearing working grid reference.`, 'warn');
|
|
4405
|
-
this.manager._persistenceWarning = persistResult;
|
|
4406
|
-
const retryResult = await this.manager.persistGrid();
|
|
4407
|
-
if (retryResult && (retryResult.skipped || retryResult.isValid === false)) {
|
|
4408
|
-
this.manager.logger.log(`[COW][PERSIST-GUARD] Retry also skipped/invalid ` +
|
|
4409
|
-
`(${retryResult.reason || 'no reason'}). Master grid in memory ` +
|
|
4410
|
-
`is ahead of disk snapshot; structural resync requested.`, 'error');
|
|
4411
|
-
if (typeof this.manager.requestStructuralGridResync === 'function') {
|
|
4412
|
-
this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
|
|
4413
|
-
await this.manager.requestStructuralGridResync('persistence guard triggered after COW batch', { persistReason: retryResult.reason || 'unknown' });
|
|
4414
|
-
}
|
|
4415
|
-
}
|
|
4416
|
-
else {
|
|
4417
|
-
delete this.manager._persistenceWarning;
|
|
4418
|
-
}
|
|
4419
|
-
}
|
|
4420
|
-
else if (this.manager._persistenceWarning) {
|
|
4421
|
-
delete this.manager._persistenceWarning;
|
|
4422
|
-
}
|
|
4423
|
-
this._metrics.batchesExecuted++;
|
|
4424
|
-
this.manager._clearWorkingGridRef();
|
|
4425
|
-
this._clearPendingBroadcasts();
|
|
4426
|
-
return { executed: true, hadRotation: true, ...batchResult };
|
|
4427
|
-
}
|
|
4428
|
-
else {
|
|
4429
|
-
// FAILURE: Working grid discarded, master unchanged
|
|
4430
|
-
this.manager.logger.log('[COW] Blockchain failed - working grid discarded, master unchanged', 'warn');
|
|
4431
|
-
this.manager._clearWorkingGridRef();
|
|
4432
|
-
this._clearPendingBroadcasts();
|
|
4433
|
-
return { executed: false, hadRotation: false, ...result };
|
|
4434
|
-
}
|
|
4435
|
-
}
|
|
4436
|
-
finally {
|
|
4437
|
-
this.manager._throwOnIllegalState = false;
|
|
4438
|
-
// Keep broadcasting true during resumeFundRecalc to skip invariant checks
|
|
4439
|
-
// that would fail due to stale accountTotals (not yet refreshed from blockchain)
|
|
4440
|
-
await this.manager.resumeFundRecalc();
|
|
4441
|
-
this.manager.stopBroadcasting();
|
|
4442
|
-
const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
|
|
4443
|
-
const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
|
|
4444
|
-
this.manager.logger.logFundsStatus(this.manager, `AFTER COW batch (created=${createCount}, cancelled=${cancelCount})`);
|
|
4445
|
-
}
|
|
4446
|
-
}
|
|
4447
|
-
catch (err) {
|
|
4448
|
-
this.manager.logger.log(`[COW] Batch transaction failed: ${err.message}`, 'error');
|
|
4449
|
-
if (err?.partialOnChainState) {
|
|
4450
|
-
this.manager.logger.log(`[COW] Non-atomic grouped execution detected (${err.groupsBroadcast}/${err.groupsTotal} groups broadcast). Local rollback cannot undo confirmed on-chain operations; next sync/reconcile will converge state.`, 'warn');
|
|
4451
|
-
}
|
|
4452
|
-
this.manager.stopBroadcasting();
|
|
4453
|
-
this.manager._clearWorkingGridRef();
|
|
4454
|
-
// BROADCAST_UNCERTAIN: the credential daemon timed out (or hit its
|
|
4455
|
-
// inner deadline) and the chain status of the planned CREATEs is
|
|
4456
|
-
// unknown. Run the reconcile-then-decide recovery path: read the
|
|
4457
|
-
// chain, match each pending broadcast by fingerprint, adopt any
|
|
4458
|
-
// chain-side matches, and discard the rest. We MUST NOT throw —
|
|
4459
|
-
// throwing would re-enter the catch loop on the next attempt and
|
|
4460
|
-
// potentially double-publish.
|
|
4461
|
-
if (err instanceof BroadcastUncertainError) {
|
|
4462
|
-
return await this._reconcileAfterUncertainBroadcast(err, opContexts);
|
|
4463
|
-
}
|
|
4464
|
-
// Handle hard abort
|
|
4465
|
-
const hardAbortResult = await this._handleBatchHardAbort(err, 'COW batch processing', operations.length);
|
|
4466
|
-
if (hardAbortResult)
|
|
4467
|
-
return hardAbortResult;
|
|
4468
|
-
// Check for stale orders — filled in the ~1.5s broadcast window after our plan was built.
|
|
4469
|
-
const staleOrderIds = new Set();
|
|
4470
|
-
const patterns = [
|
|
4471
|
-
/Limit order (1\.7\.\d+) does not exist/g,
|
|
4472
|
-
/Unable to find Object (1\.7\.\d+)/g,
|
|
4473
|
-
/object (1\.7\.\d+) (?:does not exist|not found)/gi
|
|
4474
|
-
];
|
|
4475
|
-
for (const pattern of patterns) {
|
|
4476
|
-
let m;
|
|
4477
|
-
while ((m = pattern.exec(err.message)) !== null) {
|
|
4478
|
-
staleOrderIds.add(m[1]);
|
|
4479
|
-
}
|
|
4480
|
-
}
|
|
4481
|
-
// "Cannot deduct all or more from order than order contains" means the order still
|
|
4482
|
-
// exists, but its on-chain size shrank during the broadcast window. That is not an
|
|
4483
|
-
// explicit stale/missing-order signal, so reconcile from chain instead of virtualizing.
|
|
4484
|
-
if (/Cannot deduct all or more from order than order contains/.test(err.message)) {
|
|
4485
|
-
return await this._recoverBatchSizeDrift(err, opContexts);
|
|
4486
|
-
}
|
|
4487
|
-
if (staleOrderIds.size > 0) {
|
|
4488
|
-
// Recover explicit missing-order failures without aborting the entire fill cycle.
|
|
4489
|
-
// Use the manager update path so indexes, working-grid sync, accounting, and
|
|
4490
|
-
// persistence stay coherent.
|
|
4491
|
-
return await this._recoverExplicitStaleOrders(staleOrderIds, 'cow-stale-order-cleanup');
|
|
4492
|
-
}
|
|
4493
|
-
throw err;
|
|
4494
|
-
}
|
|
4495
|
-
finally {
|
|
4496
|
-
this._batchInFlight = false;
|
|
4497
|
-
this._markGridActivity('batch end');
|
|
4498
|
-
this.manager.unlockOrders(idsToLock);
|
|
4499
|
-
if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
|
|
4500
|
-
this._scheduleFillConsumerRestart(chainOrders);
|
|
4501
|
-
}
|
|
4502
|
-
}
|
|
1395
|
+
return cowRuntime.updateOrdersOnChainBatchCOW(this, cowResult);
|
|
4503
1396
|
}
|
|
4504
|
-
/**
|
|
4505
|
-
* Process results from batch transaction execution.
|
|
4506
|
-
* Updates order state, synchronizes with chain, and deducts BTS fees.
|
|
4507
|
-
* @param {Object} result - Transaction result from executeBatch
|
|
4508
|
-
* @param {Array} opContexts - Operation context array with operation metadata (must be 1:1 with result.operation_results)
|
|
4509
|
-
* @returns {Object} Result with { executed: boolean, hadRotation: boolean }
|
|
4510
|
-
* @private
|
|
4511
|
-
*/
|
|
4512
1397
|
async _processBatchResults(result, opContexts) {
|
|
4513
|
-
|
|
4514
|
-
const { getAssetFees } = require('./order/utils/math');
|
|
4515
|
-
// IMPORTANT: Call without amount to get fee schedule fields
|
|
4516
|
-
// ({ createFee, updateFee, ... }), not proceeds projection fields.
|
|
4517
|
-
const btsFeeData = getAssetFees('BTS');
|
|
4518
|
-
let hadRotation = false;
|
|
4519
|
-
let updateOperationCount = 0;
|
|
4520
|
-
const updatesToApply = [];
|
|
4521
|
-
for (let i = 0; i < opContexts.length; i++) {
|
|
4522
|
-
const ctx = opContexts[i];
|
|
4523
|
-
const res = results[i];
|
|
4524
|
-
if (ctx.kind === 'cancel') {
|
|
4525
|
-
this.manager.logger.log(`Cancelled surplus order ${ctx.order.id} (${ctx.order.orderId})`, 'info');
|
|
4526
|
-
const oldOrder = ctx.order;
|
|
4527
|
-
const committedOrder = oldOrder?.id ? this.manager.orders.get(oldOrder.id) : null;
|
|
4528
|
-
// The COW commit already updated manager.orders. Apply ONLY the optimistic
|
|
4529
|
-
// accounting transition using pre-commit -> committed order states.
|
|
4530
|
-
if (oldOrder && committedOrder && this.manager.accountant) {
|
|
4531
|
-
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, committedOrder, 'fill-cancel', btsFeeData?.cancelFee || 0, false);
|
|
4532
|
-
}
|
|
4533
|
-
}
|
|
4534
|
-
else if (ctx.kind === 'size-update') {
|
|
4535
|
-
const oldOrder = ctx.updateInfo.partialOrder;
|
|
4536
|
-
const ord = this.manager.orders.get(oldOrder.id);
|
|
4537
|
-
// Apply optimistic accounting from pre-commit -> committed state,
|
|
4538
|
-
// including blockchain update fee deduction.
|
|
4539
|
-
if (oldOrder && ord && this.manager.accountant) {
|
|
4540
|
-
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
|
|
4541
|
-
}
|
|
4542
|
-
if (ord) {
|
|
4543
|
-
const updatedSlot = { ...ord, size: ctx.updateInfo.newSize };
|
|
4544
|
-
// Update rawOnChain cache with new integers
|
|
4545
|
-
if (ctx.finalInts) {
|
|
4546
|
-
updatedSlot.rawOnChain = {
|
|
4547
|
-
id: ord.orderId,
|
|
4548
|
-
for_sale: String(ctx.finalInts.sell),
|
|
4549
|
-
sell_price: {
|
|
4550
|
-
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
4551
|
-
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
4552
|
-
}
|
|
4553
|
-
};
|
|
4554
|
-
}
|
|
4555
|
-
updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
|
|
4556
|
-
}
|
|
4557
|
-
this.manager.logger.log(`Size update complete: ${ctx.updateInfo.partialOrder.orderId}`, 'info');
|
|
4558
|
-
updateOperationCount++;
|
|
4559
|
-
}
|
|
4560
|
-
else if (ctx.kind === 'create') {
|
|
4561
|
-
const chainOrderId = res && res[1];
|
|
4562
|
-
if (chainOrderId) {
|
|
4563
|
-
// synchronizeWithChain handles the full VIRTUAL -> ACTIVE transition
|
|
4564
|
-
// including orderId assignment and fee deduction.
|
|
4565
|
-
await this.manager.synchronizeWithChain({
|
|
4566
|
-
gridOrderId: ctx.order.id, chainOrderId, expectedType: ctx.order.type, fee: btsFeeData.createFee
|
|
4567
|
-
}, 'createOrder');
|
|
4568
|
-
// After sync, apply rawOnChain metadata if available
|
|
4569
|
-
if (ctx.finalInts) {
|
|
4570
|
-
const syncedOrder = this.manager.orders.get(ctx.order.id);
|
|
4571
|
-
if (syncedOrder) {
|
|
4572
|
-
updatesToApply.push({
|
|
4573
|
-
order: {
|
|
4574
|
-
...syncedOrder,
|
|
4575
|
-
rawOnChain: {
|
|
4576
|
-
id: chainOrderId,
|
|
4577
|
-
for_sale: String(ctx.finalInts.sell),
|
|
4578
|
-
sell_price: {
|
|
4579
|
-
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
4580
|
-
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
4581
|
-
}
|
|
4582
|
-
}
|
|
4583
|
-
},
|
|
4584
|
-
context: 'post-placement-metadata'
|
|
4585
|
-
});
|
|
4586
|
-
}
|
|
4587
|
-
}
|
|
4588
|
-
this.manager.logger.log(`Placed ${ctx.order.type} order ${ctx.order.id} -> ${chainOrderId}`, 'info');
|
|
4589
|
-
}
|
|
4590
|
-
else {
|
|
4591
|
-
const fingerprint = [
|
|
4592
|
-
`type=${ctx.order.type || 'unknown'}`,
|
|
4593
|
-
`price=${Format.formatPrice6(ctx.order.price)}`,
|
|
4594
|
-
`size=${Format.formatAmount(ctx.order.size)}`
|
|
4595
|
-
].join(',');
|
|
4596
|
-
this.manager.logger.log(`[COW] CRITICAL: Create op for slot ${ctx.order.id} (type=${ctx.order.type}) ` +
|
|
4597
|
-
`returned no chainOrderId. Identify any orphaned on-chain order by local fingerprint ` +
|
|
4598
|
-
`${fingerprint} before cancelling.`, 'error');
|
|
4599
|
-
}
|
|
4600
|
-
}
|
|
4601
|
-
else if (ctx.kind === 'rotation') {
|
|
4602
|
-
hadRotation = true;
|
|
4603
|
-
const { rotation } = ctx;
|
|
4604
|
-
const { oldOrder, newPrice, newGridId, newSize, type } = rotation;
|
|
4605
|
-
if (!newGridId) {
|
|
4606
|
-
// Size correction only
|
|
4607
|
-
const ord = this.manager.orders.get(oldOrder.id || rotation.id);
|
|
4608
|
-
// Apply optimistic accounting from pre-commit -> committed state,
|
|
4609
|
-
// including blockchain update fee deduction.
|
|
4610
|
-
if (oldOrder && ord && this.manager.accountant) {
|
|
4611
|
-
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
|
|
4612
|
-
}
|
|
4613
|
-
if (ord) {
|
|
4614
|
-
const updatedSlot = { ...ord, size: newSize };
|
|
4615
|
-
// Update rawOnChain cache with new integers
|
|
4616
|
-
if (ctx.finalInts) {
|
|
4617
|
-
updatedSlot.rawOnChain = {
|
|
4618
|
-
id: ord.orderId,
|
|
4619
|
-
for_sale: String(ctx.finalInts.sell),
|
|
4620
|
-
sell_price: {
|
|
4621
|
-
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
4622
|
-
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
4623
|
-
}
|
|
4624
|
-
};
|
|
4625
|
-
}
|
|
4626
|
-
updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
|
|
4627
|
-
}
|
|
4628
|
-
updateOperationCount++;
|
|
4629
|
-
continue;
|
|
4630
|
-
}
|
|
4631
|
-
// Full rotation: old slot was virtualized in the committed working grid.
|
|
4632
|
-
// Activate the destination slot with the existing on-chain orderId.
|
|
4633
|
-
const slot = this.manager.orders.get(newGridId);
|
|
4634
|
-
if (!slot) {
|
|
4635
|
-
this.manager.logger.log(`[ROTATION] Destination slot ${newGridId} missing from master grid after COW commit - skipping activation, sync will reconcile`, 'error');
|
|
4636
|
-
// Still clear source orderId to prevent a stale chain reference persisting
|
|
4637
|
-
if (oldOrder?.id && oldOrder.id !== newGridId) {
|
|
4638
|
-
const staleSource = this.manager.orders.get(oldOrder.id);
|
|
4639
|
-
if (staleSource?.orderId) {
|
|
4640
|
-
updatesToApply.push({
|
|
4641
|
-
order: { ...staleSource, state: ORDER_STATES.VIRTUAL, orderId: null, rawOnChain: null },
|
|
4642
|
-
context: 'post-rotation-source-clear'
|
|
4643
|
-
});
|
|
4644
|
-
}
|
|
4645
|
-
}
|
|
4646
|
-
continue;
|
|
4647
|
-
}
|
|
4648
|
-
const updatedSlot = {
|
|
4649
|
-
...slot,
|
|
4650
|
-
id: newGridId,
|
|
4651
|
-
type,
|
|
4652
|
-
size: newSize,
|
|
4653
|
-
price: newPrice,
|
|
4654
|
-
state: ORDER_STATES.ACTIVE,
|
|
4655
|
-
orderId: oldOrder?.orderId || slot.orderId || null
|
|
4656
|
-
};
|
|
4657
|
-
if (ctx.finalInts) {
|
|
4658
|
-
updatedSlot.rawOnChain = {
|
|
4659
|
-
id: updatedSlot.orderId,
|
|
4660
|
-
for_sale: String(ctx.finalInts.sell),
|
|
4661
|
-
sell_price: {
|
|
4662
|
-
base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
|
|
4663
|
-
quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
|
|
4664
|
-
}
|
|
4665
|
-
};
|
|
4666
|
-
}
|
|
4667
|
-
if (oldOrder && updatedSlot && this.manager.accountant) {
|
|
4668
|
-
await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, updatedSlot, 'order-update', btsFeeData.updateFee, false);
|
|
4669
|
-
}
|
|
4670
|
-
// Ensure rotation source slot is cleared in master state.
|
|
4671
|
-
// COW projection may keep source slots ACTIVE when target keeps a non-zero
|
|
4672
|
-
// virtual size, but after a successful on-chain rotation the source orderId
|
|
4673
|
-
// must no longer remain attached to the old slot.
|
|
4674
|
-
if (oldOrder?.id && oldOrder.id !== newGridId) {
|
|
4675
|
-
const currentSource = this.manager.orders.get(oldOrder.id);
|
|
4676
|
-
if (currentSource && currentSource.orderId) {
|
|
4677
|
-
updatesToApply.push({
|
|
4678
|
-
order: {
|
|
4679
|
-
...currentSource,
|
|
4680
|
-
state: ORDER_STATES.VIRTUAL,
|
|
4681
|
-
orderId: null,
|
|
4682
|
-
rawOnChain: null
|
|
4683
|
-
},
|
|
4684
|
-
context: 'post-rotation-source-clear'
|
|
4685
|
-
});
|
|
4686
|
-
}
|
|
4687
|
-
}
|
|
4688
|
-
updatesToApply.push({ order: updatedSlot, context: 'post-rotation-metadata' });
|
|
4689
|
-
}
|
|
4690
|
-
}
|
|
4691
|
-
// Apply all collected updates in a single batch
|
|
4692
|
-
if (updatesToApply.length > 0) {
|
|
4693
|
-
await this.manager.applyGridUpdateBatch(updatesToApply.map(u => u.order), 'batch-results-process', { skipAccounting: true });
|
|
4694
|
-
}
|
|
4695
|
-
return {
|
|
4696
|
-
executed: true,
|
|
4697
|
-
hadRotation,
|
|
4698
|
-
updateOperationCount
|
|
4699
|
-
};
|
|
1398
|
+
return cowRuntime.processBatchResults(this, result, opContexts);
|
|
4700
1399
|
}
|
|
4701
1400
|
/**
|
|
4702
1401
|
* Perform grid recalculation triggered by trigger file.
|
|
@@ -4937,89 +1636,17 @@ class DEXBot {
|
|
|
4937
1636
|
}
|
|
4938
1637
|
}
|
|
4939
1638
|
async requestGridReset(reason = 'structural change', options = {}) {
|
|
4940
|
-
|
|
4941
|
-
return { skipped: true, reason: 'grid resync unavailable' };
|
|
4942
|
-
}
|
|
4943
|
-
const message = reason ? `[CR-RESET] ${reason}` : '[CR-RESET] grid reset requested';
|
|
4944
|
-
this._log(`${message}; rebuilding grid from fresh on-chain state`, 'info');
|
|
4945
|
-
const resetOptions = {
|
|
4946
|
-
...options,
|
|
4947
|
-
refreshCenterPrice: options.refreshCenterPrice !== false,
|
|
4948
|
-
};
|
|
4949
|
-
// Skip acquire if no lock exists (no fill processing to serialize against)
|
|
4950
|
-
// or if the caller already holds the lock (re-entrant). In the latter case
|
|
4951
|
-
// the lock's isReentrant() check prevents queueing and runs inline.
|
|
4952
|
-
// Otherwise, wait on the queue — this is the intended path for callers
|
|
4953
|
-
// outside the fill-processing context that need exclusive access.
|
|
4954
|
-
if (!this.manager._fillProcessingLock || this.manager._fillProcessingLock.isReentrant()) {
|
|
4955
|
-
return this._performGridResync(resetOptions);
|
|
4956
|
-
}
|
|
4957
|
-
return this.manager._fillProcessingLock.acquire(async () => this._performGridResync(resetOptions));
|
|
1639
|
+
return DexbotMaintenanceRuntime.requestGridReset(this, reason, options);
|
|
4958
1640
|
}
|
|
4959
1641
|
_wireStructuralGridResyncRequest() {
|
|
4960
|
-
|
|
4961
|
-
return;
|
|
4962
|
-
this.manager.requestStructuralGridResync = async (reason = 'structural recovery', details = {}) => {
|
|
4963
|
-
if (this._shuttingDown) {
|
|
4964
|
-
return { skipped: true, reason: 'shutting down' };
|
|
4965
|
-
}
|
|
4966
|
-
if (this._structuralGridResyncRunning || this._structuralGridResyncTimer) {
|
|
4967
|
-
return { skipped: true, reason: 'structural grid resync already scheduled' };
|
|
4968
|
-
}
|
|
4969
|
-
const unmatchedCount = Array.isArray(details?.unmatchedChainOrders)
|
|
4970
|
-
? details.unmatchedChainOrders.length
|
|
4971
|
-
: 0;
|
|
4972
|
-
this._structuralGridResyncTimer = setTimeout(async () => {
|
|
4973
|
-
this._structuralGridResyncTimer = null;
|
|
4974
|
-
if (this._shuttingDown)
|
|
4975
|
-
return;
|
|
4976
|
-
this._structuralGridResyncRunning = true;
|
|
4977
|
-
try {
|
|
4978
|
-
// Try the lighter persisted-grid reload before full reset.
|
|
4979
|
-
const persistedResult = await this._recoverFromPersistedGrid();
|
|
4980
|
-
if (persistedResult.success) {
|
|
4981
|
-
if (this.manager?._recoveryState) {
|
|
4982
|
-
this.manager._recoveryState = { ...this.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
|
|
4983
|
-
}
|
|
4984
|
-
return;
|
|
4985
|
-
}
|
|
4986
|
-
const suffix = unmatchedCount > 0 ? ` (${unmatchedCount} unmatched chain order(s))` : '';
|
|
4987
|
-
this._warn(`[RECOVERY] Running structural full grid resync for ${reason}${suffix}`);
|
|
4988
|
-
const resetResult = await this.requestGridReset('rms_structural_grid_resync', {
|
|
4989
|
-
refreshCenterPrice: true,
|
|
4990
|
-
});
|
|
4991
|
-
if (resetResult && this.manager?._recoveryState) {
|
|
4992
|
-
this.manager._recoveryState = { ...this.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
|
|
4993
|
-
}
|
|
4994
|
-
}
|
|
4995
|
-
catch (err) {
|
|
4996
|
-
this._warn(`[RECOVERY] Structural full grid resync failed: ${err.message}`);
|
|
4997
|
-
}
|
|
4998
|
-
finally {
|
|
4999
|
-
this._structuralGridResyncRunning = false;
|
|
5000
|
-
if (this.manager?._recoveryState) {
|
|
5001
|
-
this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: false };
|
|
5002
|
-
}
|
|
5003
|
-
}
|
|
5004
|
-
}, 0);
|
|
5005
|
-
return { scheduled: true };
|
|
5006
|
-
};
|
|
1642
|
+
return DexbotMaintenanceRuntime.wireStructuralGridResyncRequest(this);
|
|
5007
1643
|
}
|
|
5008
1644
|
/**
|
|
5009
1645
|
* Get current metrics for monitoring and debugging.
|
|
5010
1646
|
* @returns {Object} Metrics snapshot
|
|
5011
1647
|
*/
|
|
5012
1648
|
getMetrics() {
|
|
5013
|
-
|
|
5014
|
-
return {
|
|
5015
|
-
...this._metrics,
|
|
5016
|
-
queueDepth: this._incomingFillQueue.length,
|
|
5017
|
-
fillProcessingLockActive: this.manager?._fillProcessingLock?.isLocked() || false,
|
|
5018
|
-
divergenceLockActive: this.manager?._divergenceLock?.isLocked() || false,
|
|
5019
|
-
shadowLocksActive: this.manager?.shadowOrderIds?.size || 0,
|
|
5020
|
-
recoveryExhaustedAt: this.manager?._recoveryExhaustedAt || null,
|
|
5021
|
-
recentFillsTracked: this._recentlyProcessedFills.size
|
|
5022
|
-
};
|
|
1649
|
+
return DexbotMaintenanceRuntime.getMetrics(this);
|
|
5023
1650
|
}
|
|
5024
1651
|
/**
|
|
5025
1652
|
* Execute grid maintenance checks in strict order with pipeline consensus.
|
|
@@ -5074,41 +1701,7 @@ class DEXBot {
|
|
|
5074
1701
|
* @private
|
|
5075
1702
|
*/
|
|
5076
1703
|
async _runDustHealthCheck() {
|
|
5077
|
-
|
|
5078
|
-
return;
|
|
5079
|
-
try {
|
|
5080
|
-
const health = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan?.bind(this));
|
|
5081
|
-
const buyDust = health.buyDustOrders || [];
|
|
5082
|
-
const sellDust = health.sellDustOrders || [];
|
|
5083
|
-
const totalDust = buyDust.length + sellDust.length;
|
|
5084
|
-
if (totalDust > 0) {
|
|
5085
|
-
this._log(`[DUST] Health check: ${totalDust} dust order(s) (buy=${buyDust.length}, sell=${sellDust.length})`);
|
|
5086
|
-
const lock = this.manager._fillProcessingLock;
|
|
5087
|
-
if (lock && typeof lock.acquire === 'function') {
|
|
5088
|
-
await lock.acquire(async () => {
|
|
5089
|
-
await this._cancelDustOrders({
|
|
5090
|
-
buy: health.buyDustOrders,
|
|
5091
|
-
sell: health.sellDustOrders,
|
|
5092
|
-
});
|
|
5093
|
-
}, { timeout: TIMING.DUST_CANCEL_TIMEOUT_MS });
|
|
5094
|
-
}
|
|
5095
|
-
else {
|
|
5096
|
-
this._warn('[DUST] Fill lock unavailable — cancelling dust without lock (potential race)');
|
|
5097
|
-
await this._cancelDustOrders({
|
|
5098
|
-
buy: health.buyDustOrders,
|
|
5099
|
-
sell: health.sellDustOrders,
|
|
5100
|
-
});
|
|
5101
|
-
}
|
|
5102
|
-
}
|
|
5103
|
-
}
|
|
5104
|
-
catch (err) {
|
|
5105
|
-
if (err?.message?.includes('Lock acquisition timeout')) {
|
|
5106
|
-
this._warn('[DUST] Lock busy, skipping dust cancel this cycle (retry in 5 min)');
|
|
5107
|
-
}
|
|
5108
|
-
else {
|
|
5109
|
-
this._warn(`[DUST] Health check error (retry in 5 min): ${err?.message || err}`);
|
|
5110
|
-
}
|
|
5111
|
-
}
|
|
1704
|
+
return DexbotMaintenanceRuntime.runDustHealthCheck(this);
|
|
5112
1705
|
}
|
|
5113
1706
|
/**
|
|
5114
1707
|
* Set up the periodic dust health check interval.
|
|
@@ -5118,12 +1711,7 @@ class DEXBot {
|
|
|
5118
1711
|
* @private
|
|
5119
1712
|
*/
|
|
5120
1713
|
_setupDustHealthCheckInterval() {
|
|
5121
|
-
|
|
5122
|
-
this._runDustHealthCheck();
|
|
5123
|
-
}, TIMING.DUST_HEALTH_CHECK_INTERVAL_MS);
|
|
5124
|
-
if (typeof this._dustHealthCheckTimer?.unref === 'function') {
|
|
5125
|
-
this._dustHealthCheckTimer.unref();
|
|
5126
|
-
}
|
|
1714
|
+
return DexbotMaintenanceRuntime.setupDustHealthCheckInterval(this);
|
|
5127
1715
|
}
|
|
5128
1716
|
/**
|
|
5129
1717
|
* Perform grid maintenance: fund thresholds, spread condition, grid health, divergence.
|