dexbot 1.3.1 → 1.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +7 -2
  2. package/dist/credential-daemon.d.ts.map +1 -1
  3. package/dist/credential-daemon.js +53 -13
  4. package/dist/credential-daemon.js.map +1 -1
  5. package/dist/modules/bitshares-native/serial/types.js +1 -1
  6. package/dist/modules/bitshares-native/serial/types.js.map +1 -1
  7. package/dist/modules/chain_keys.js +1 -1
  8. package/dist/modules/chain_keys.js.map +1 -1
  9. package/dist/modules/constants.d.ts +4 -0
  10. package/dist/modules/constants.d.ts.map +1 -1
  11. package/dist/modules/constants.js +19 -0
  12. package/dist/modules/constants.js.map +1 -1
  13. package/dist/modules/credential_policy.js +1 -1
  14. package/dist/modules/credential_policy.js.map +1 -1
  15. package/dist/modules/dexbot_class.d.ts +54 -175
  16. package/dist/modules/dexbot_class.d.ts.map +1 -1
  17. package/dist/modules/dexbot_class.js +99 -3469
  18. package/dist/modules/dexbot_class.js.map +1 -1
  19. package/dist/modules/dexbot_cow_runtime.d.ts +319 -0
  20. package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -0
  21. package/dist/modules/dexbot_cow_runtime.js +1764 -0
  22. package/dist/modules/dexbot_cow_runtime.js.map +1 -0
  23. package/dist/modules/dexbot_fill_runtime.d.ts +47 -0
  24. package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
  25. package/dist/modules/dexbot_fill_runtime.js +607 -3
  26. package/dist/modules/dexbot_fill_runtime.js.map +1 -1
  27. package/dist/modules/dexbot_maintenance_runtime.d.ts +69 -0
  28. package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
  29. package/dist/modules/dexbot_maintenance_runtime.js +271 -29
  30. package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
  31. package/dist/modules/dexbot_startup_runtime.d.ts +30 -0
  32. package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -0
  33. package/dist/modules/dexbot_startup_runtime.js +539 -0
  34. package/dist/modules/dexbot_startup_runtime.js.map +1 -0
  35. package/dist/modules/dexbot_state_recovery.d.ts +128 -0
  36. package/dist/modules/dexbot_state_recovery.d.ts.map +1 -0
  37. package/dist/modules/dexbot_state_recovery.js +397 -0
  38. package/dist/modules/dexbot_state_recovery.js.map +1 -0
  39. package/dist/modules/fund_registry.d.ts.map +1 -1
  40. package/dist/modules/fund_registry.js +0 -1
  41. package/dist/modules/fund_registry.js.map +1 -1
  42. package/dist/modules/general_settings.d.ts.map +1 -1
  43. package/dist/modules/general_settings.js +0 -1
  44. package/dist/modules/general_settings.js.map +1 -1
  45. package/dist/modules/key_store.d.ts.map +1 -1
  46. package/dist/modules/key_store.js +1 -2
  47. package/dist/modules/key_store.js.map +1 -1
  48. package/dist/modules/launcher/market_adapter_runtime.d.ts.map +1 -1
  49. package/dist/modules/launcher/market_adapter_runtime.js +0 -1
  50. package/dist/modules/launcher/market_adapter_runtime.js.map +1 -1
  51. package/dist/modules/launcher/market_adapter_watchdog.d.ts.map +1 -1
  52. package/dist/modules/launcher/market_adapter_watchdog.js +1 -2
  53. package/dist/modules/launcher/market_adapter_watchdog.js.map +1 -1
  54. package/dist/modules/order/grid.d.ts.map +1 -1
  55. package/dist/modules/order/grid.js +11 -4
  56. package/dist/modules/order/grid.js.map +1 -1
  57. package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
  58. package/dist/modules/order/grid_reconcile.js +0 -1
  59. package/dist/modules/order/grid_reconcile.js.map +1 -1
  60. package/dist/modules/order/grid_reconcile_internal.js +2 -2
  61. package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
  62. package/dist/modules/order/utils/math.js +1 -1
  63. package/dist/modules/order/utils/math.js.map +1 -1
  64. package/dist/modules/order/utils/system.js +1 -1
  65. package/dist/modules/order/utils/system.js.map +1 -1
  66. package/dist/modules/order/utils/validate.d.ts.map +1 -1
  67. package/dist/modules/order/utils/validate.js +0 -1
  68. package/dist/modules/order/utils/validate.js.map +1 -1
  69. package/dist/modules/paths.d.ts +2 -0
  70. package/dist/modules/paths.d.ts.map +1 -1
  71. package/dist/modules/paths.js +13 -2
  72. package/dist/modules/paths.js.map +1 -1
  73. package/dist/scripts/verify-browser-bundle.js +25 -1
  74. package/dist/scripts/verify-browser-bundle.js.map +1 -1
  75. package/dist/unlock.js +2 -2
  76. package/dist/unlock.js.map +1 -1
  77. package/package.json +7 -2
@@ -53,33 +53,26 @@
53
53
  *
54
54
  * ===============================================================================
55
55
  */
56
- const { path } = require('./path_api');
57
- const { BitShares, waitForConnected, onReconnect: registerReconnectHook } = require('./bitshares_client');
58
- const { getStorage } = require('./storage');
59
- const storage = getStorage();
56
+ const { BitShares } = require('./bitshares_client');
60
57
  const chainKeys = require('./chain_keys');
61
- const { getKeyStore } = require('./key_store');
62
58
  const chainOrders = require('./chain_orders');
63
59
  const fundRegistry = require('./fund_registry');
64
- const { BroadcastUncertainError } = require('./dexbot_credential_client');
65
- const { OrderManager, grid: Grid } = require('./order');
66
- const { retryPersistenceIfNeeded, initializeFeeCache, } = require('./order/utils/system');
67
- const { hasExecutableActions, validateCreateTargetSlots } = require('./order/utils/validate');
68
- const { buildCreateOrderArgs, buildCreateOpFingerprint, virtualizeOrder, correctAllPriceMismatches, convertToSpreadPlaceholder, buildOutsideInPairGroups, extractBatchOperationResults, buildFillKey, formatUnmatchedChainOrder } = require('./order/utils/order');
69
- const { validateOrderSize } = require('./order/utils/math');
70
60
  const { ProcessedFillStore, PROCESSED_FILL_PERSISTENCE_MODES } = require('./order/processed_fill_store');
71
61
  const DexbotFillRuntime = require('./dexbot_fill_runtime');
72
62
  const DexbotMaintenanceRuntime = require('./dexbot_maintenance_runtime');
63
+ const DexbotStateRecovery = require('./dexbot_state_recovery');
64
+ const DexbotStartupRuntime = require('./dexbot_startup_runtime');
73
65
  const CreditRuntime = require('./credit_runtime');
74
- const { ORDER_STATES, ORDER_TYPES, REBALANCE_STATES, COW_ACTIONS, TIMING, PIPELINE_TIMING, GRID_LIMITS, MAINTENANCE, FILL_PROCESSING, DAEMON_CODES, } = require('./constants');
75
- const { PATHS, getRecalculateTriggerFile } = require('./paths');
76
- const { attemptResumePersistedGridByPriceMatch, decideStartupGridAction, reconcileGridOrders } = require('./order/grid_reconcile');
77
- const { AccountOrders } = require('./account_orders');
78
- const { parseJsonWithComments } = require('./order/utils/system');
79
- const { cloneWeightDistribution } = require('./order/utils/math');
80
- const { normalizeBotEntry } = require('./bot_settings');
66
+ const { TIMING, FILL_PROCESSING, DAEMON_CODES, } = require('./constants');
67
+ const { PATHS } = require('./paths');
81
68
  const Format = require('./order/format');
82
- const { resolveBotRuntimeSettings } = require('./runtime_settings');
69
+ const cowRuntime = require('./dexbot_cow_runtime');
70
+ function waitForConnected(...args) { return require('./bitshares_client').waitForConnected(...args); }
71
+ function getKeyStore(...args) { return require('./key_store').getKeyStore(...args); }
72
+ function hasExecutableActions(...args) { return require('./order/utils/validate').hasExecutableActions(...args); }
73
+ function getRecalculateTriggerFile(...args) { return require('./paths').getRecalculateTriggerFile(...args); }
74
+ function cloneWeightDistribution(...args) { return require('./order/utils/math').cloneWeightDistribution(...args); }
75
+ function resolveBotRuntimeSettings(...args) { return require('./runtime_settings').resolveBotRuntimeSettings(...args); }
83
76
  const PROFILES_BOTS_FILE = PATHS.PROFILES.BOTS_JSON;
84
77
  const PROFILES_DIR = PATHS.PROFILES_DIR;
85
78
  class DEXBot {
@@ -142,6 +135,7 @@ class DEXBot {
142
135
  _ghostOrderCancelAttempted;
143
136
  _dustHealthCheckTimer;
144
137
  _lastBroadcastHeartbeatAt;
138
+ _lastDeferredDustCount;
145
139
  _currentBatchId;
146
140
  /**
147
141
  * Create a new DEXBot instance
@@ -222,6 +216,7 @@ class DEXBot {
222
216
  this._targetedDriftSyncCooldownMs = this.config.timing.TARGETED_DRIFT_SYNC_COOLDOWN_MS;
223
217
  this._maintenanceCooldownCycles = 0;
224
218
  this._lastGridActivityAt = 0;
219
+ this._lastDeferredDustCount = 0;
225
220
  this._currentCycleId = 0;
226
221
  this._autoCancelOrphanCycleMarker = null;
227
222
  this._autoCancelOrphanSubCount = 0;
@@ -322,21 +317,7 @@ class DEXBot {
322
317
  * @private
323
318
  */
324
319
  async _persistAndRecoverIfNeeded() {
325
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
326
- const validation = await this.manager.persistGrid();
327
- if (!validation.isValid) {
328
- this._warn(`Startup validation failed: ${validation.reason}. Triggering immediate recovery...`);
329
- // Trigger centralized recovery (Hard Reset)
330
- const recoveryValidation = await this.manager.accountant._performStateRecovery(this.manager);
331
- if (recoveryValidation.isValid) {
332
- this._log(`✓ Startup recovery successful. Persistent state restored.`);
333
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
334
- await this.manager.persistGrid();
335
- }
336
- else {
337
- this._warn(`Startup recovery failed: ${recoveryValidation.reason}. Bot proceeding with caution.`);
338
- }
339
- }
320
+ return DexbotStateRecovery.persistAndRecoverIfNeeded(this);
340
321
  }
341
322
  /**
342
323
  * Snapshot the recently queued fill keys as a plain object for crash-durable persistence.
@@ -346,42 +327,14 @@ class DEXBot {
346
327
  * @returns {Record<string, number>}
347
328
  */
348
329
  _getRecentFillKeysSnapshot() {
349
- const snapshot = {};
350
- const now = Date.now();
351
- // 1. Collect live keys from the in-memory map
352
- for (const [key, timestamp] of this._recentlyQueuedFills) {
353
- if (now - Number(timestamp) < this._fillDedupeWindowMs) {
354
- snapshot[key] = Number(timestamp);
355
- }
356
- }
357
- // 2. Merge with the previous snapshot — carry forward any keys that
358
- // were present in a prior persist cycle, are still within the
359
- // dedup window, but have been evicted from _recentlyQueuedFills
360
- // (e.g. by TTL-based pruning between two persistGrid calls).
361
- // This prevents a crash + restart from losing keys that were
362
- // processed but not yet persisted.
363
- if (this.manager?._recentFillKeysSnapshot) {
364
- for (const [key, timestamp] of Object.entries(this.manager._recentFillKeysSnapshot)) {
365
- if (!(key in snapshot) && now - Number(timestamp) < this._fillDedupeWindowMs) {
366
- snapshot[key] = Number(timestamp);
367
- }
368
- }
369
- }
370
- return snapshot;
330
+ return DexbotStateRecovery.getRecentFillKeysSnapshot(this);
371
331
  }
372
332
  /**
373
333
  * Get current pipeline signal state for congestion checks.
374
334
  * @returns {{incomingFillQueueLength: number, shadowLocks: number, batchInFlight: boolean, recoveryInFlight: boolean, broadcasting: boolean}}
375
335
  */
376
336
  _getPipelineSignals() {
377
- this.manager?._cleanExpiredLocks?.();
378
- return {
379
- incomingFillQueueLength: this._incomingFillQueue.length,
380
- shadowLocks: this.manager?.shadowOrderIds?.size || 0,
381
- batchInFlight: this._batchInFlight,
382
- recoveryInFlight: this._recoverySyncInFlight,
383
- broadcasting: this.manager?.isBroadcastingActive?.() || false
384
- };
337
+ return DexbotMaintenanceRuntime.getPipelineSignals(this);
385
338
  }
386
339
  /**
387
340
  * Mark that grid activity occurred (updates idle timer).
@@ -389,8 +342,7 @@ class DEXBot {
389
342
  * @returns {void}
390
343
  */
391
344
  _markGridActivity(reason = 'activity') {
392
- this._lastGridActivityAt = Date.now();
393
- this.manager?.logger?.log?.(`[MAINT-IDLE] Activity observed: ${reason}`, 'debug');
345
+ return DexbotMaintenanceRuntime.markGridActivity(this, reason);
394
346
  }
395
347
  /**
396
348
  * Trigger a full state recovery sync (fetch chain + sync from open orders + persist).
@@ -398,25 +350,7 @@ class DEXBot {
398
350
  * @returns {Promise<void>}
399
351
  */
400
352
  async _triggerStateRecoverySync(reason = 'state recovery sync') {
401
- if (!this.manager)
402
- return;
403
- if (this._recoverySyncInFlight) {
404
- this.manager.logger.log(`[RECOVERY] Skipping duplicate recovery request: ${reason}`, 'warn');
405
- return;
406
- }
407
- this._recoverySyncInFlight = true;
408
- try {
409
- this.manager.logger.log(`Triggering state recovery sync (${reason})...`, 'info');
410
- await this.manager.fetchAccountTotals(this.accountId);
411
- const openOrders = await chainOrders.readOpenOrders(this.accountId);
412
- await this.manager.syncFromOpenOrders(openOrders, { skipAccounting: true });
413
- if (typeof this.manager.persistGrid === 'function') {
414
- await this.manager.persistGrid();
415
- }
416
- }
417
- finally {
418
- this._recoverySyncInFlight = false;
419
- }
353
+ return DexbotStateRecovery.triggerStateRecoverySync(this, reason);
420
354
  }
421
355
  /**
422
356
  * Abort the current flow if an illegal state signal was raised.
@@ -424,14 +358,7 @@ class DEXBot {
424
358
  * @returns {Promise<boolean>} True if flow was aborted
425
359
  */
426
360
  async _abortFlowIfIllegalState(flowContext) {
427
- const illegalSignal = this.manager?.consumeIllegalStateSignal?.();
428
- if (!illegalSignal) {
429
- return false;
430
- }
431
- this.manager.logger.log(`[HARD-ABORT] ${flowContext} aborted due to illegal state (${illegalSignal.context}): ${illegalSignal.message}`, 'error');
432
- await this._triggerStateRecoverySync(`hard-abort ${flowContext}`);
433
- this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
434
- return true;
361
+ return DexbotStateRecovery.abortFlowIfIllegalState(this, flowContext);
435
362
  }
436
363
  /**
437
364
  * Handle a hard abort from batch processing due to illegal state or accounting failure.
@@ -441,24 +368,7 @@ class DEXBot {
441
368
  * @returns {Promise<Object>} Abort result object
442
369
  */
443
370
  async _handleBatchHardAbort(err, phase = 'batch processing', opsCount = 0) {
444
- const baseResult = { executed: false, hadRotation: false };
445
- const opsInfo = opsCount > 0 ? ` with ${opsCount} ops` : '';
446
- if (err?.code === 'ILLEGAL_ORDER_STATE') {
447
- const illegalSignal = this.manager.consumeIllegalStateSignal?.();
448
- await this._triggerStateRecoverySync(illegalSignal?.message || `illegal order state during ${phase}${opsInfo}`);
449
- this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
450
- return { ...baseResult, abortedForIllegalState: true };
451
- }
452
- if (err?.code === 'ACCOUNTING_COMMITMENT_FAILED') {
453
- const accountingSignal = this.manager.consumeAccountingFailureSignal?.();
454
- const reason = accountingSignal
455
- ? `accounting lock failure (${accountingSignal.side} ${Format.formatAmount8(accountingSignal.amount)}) during ${accountingSignal.context}`
456
- : `accounting commitment lock failure during ${phase}${opsInfo}`;
457
- await this._triggerStateRecoverySync(reason);
458
- this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
459
- return { ...baseResult, abortedForAccountingFailure: true };
460
- }
461
- return null;
371
+ return DexbotStateRecovery.handleBatchHardAbort(this, err, phase, opsCount);
462
372
  }
463
373
  /**
464
374
  * Apply recoverable grid updates (order virtualisation) after a batch failure.
@@ -467,30 +377,7 @@ class DEXBot {
467
377
  * @returns {Promise<number>} Number of updates applied
468
378
  */
469
379
  async _applyRecoverableGridUpdates(updates, context = 'recoverable-grid-update') {
470
- if (!this.manager || !Array.isArray(updates) || updates.length === 0) {
471
- return 0;
472
- }
473
- let applied;
474
- if (typeof this.manager.applyGridUpdateBatch === 'function') {
475
- await this.manager.applyGridUpdateBatch(updates, context);
476
- applied = updates.length;
477
- }
478
- else {
479
- applied = 0;
480
- for (const update of updates) {
481
- if (typeof this.manager._updateOrder !== 'function')
482
- break;
483
- await this.manager._updateOrder(update, context);
484
- applied++;
485
- }
486
- }
487
- // Persist master grid mutations applied outside COW (stale-order
488
- // virtualization, size-drift corrections). These run in the COW catch
489
- // handler where the success-path persistGrid is never reached.
490
- if (applied > 0 && typeof this.manager.persistGrid === 'function') {
491
- await this.manager.persistGrid();
492
- }
493
- return applied;
380
+ return DexbotStateRecovery.applyRecoverableGridUpdates(this, updates, context);
494
381
  }
495
382
  /**
496
383
  * Recover from explicit stale order errors by virtualizing affected grid slots.
@@ -499,36 +386,7 @@ class DEXBot {
499
386
  * @returns {Promise<{executed: boolean, hadRotation: boolean, stale: boolean, recoveredByVirtualization?: boolean}>}
500
387
  */
501
388
  async _recoverExplicitStaleOrders(staleOrderIds, reason = 'stale order cleanup') {
502
- const staleIds = Array.from(staleOrderIds || []).filter(Boolean);
503
- if (staleIds.length === 0) {
504
- return { executed: false, hadRotation: false, stale: false };
505
- }
506
- this.manager.logger.log(`[COW] Stale order(s) detected: ${staleIds.join(', ')}. Applying targeted cleanup.`, 'warn');
507
- const updates = [];
508
- for (const [, gridOrder] of this.manager.orders.entries()) {
509
- if (!gridOrder?.orderId || !staleOrderIds.has(gridOrder.orderId))
510
- continue;
511
- this._staleCleanedOrderIds.set(gridOrder.orderId, Date.now());
512
- updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
513
- }
514
- // Register any stale IDs that had no matching grid slot
515
- for (const orderId of staleIds) {
516
- if (!this._staleCleanedOrderIds.has(orderId)) {
517
- this._staleCleanedOrderIds.set(orderId, Date.now());
518
- }
519
- }
520
- if (updates.length > 0) {
521
- await this._applyRecoverableGridUpdates(updates, reason);
522
- }
523
- else {
524
- this.manager.logger.log(`[COW] No local grid slot matched stale order cleanup request (${staleIds.join(', ')}).`, 'debug');
525
- }
526
- return {
527
- executed: false,
528
- hadRotation: false,
529
- stale: true,
530
- recoveredByVirtualization: updates.length > 0
531
- };
389
+ return DexbotStateRecovery.recoverExplicitStaleOrders(this, staleOrderIds, reason);
532
390
  }
533
391
  /**
534
392
  * Recover from on-chain size drift detected during batch broadcast.
@@ -536,32 +394,7 @@ class DEXBot {
536
394
  * @returns {Promise<{executed: boolean, hadRotation: boolean, recoveredBySync: boolean, reason: string}>}
537
395
  */
538
396
  async _recoverBatchSizeDrift(err, opContexts = []) {
539
- // Try a targeted fix first: extract the affected order IDs from the
540
- // operation contexts and correct them directly from chain. This
541
- // avoids a full state recovery sync in the common single-order case.
542
- const affectedOrderIds = this._extractSizeDriftOrderIds(opContexts);
543
- if (affectedOrderIds.length > 0) {
544
- this.manager.logger.log(`[COW] Targeted size-drift repair for ${affectedOrderIds.length} order(s): ${affectedOrderIds.join(', ')}`, 'debug');
545
- const repaired = await this._targetedOrderRepair(affectedOrderIds);
546
- if (repaired) {
547
- return {
548
- executed: false,
549
- hadRotation: false,
550
- recoveredBySync: true,
551
- reason: 'ORDER_SIZE_DRIFT_TARGETED'
552
- };
553
- }
554
- this.manager.logger.log('[COW] Targeted repair failed, falling back to full state recovery sync.', 'warn');
555
- }
556
- const reason = `recoverable size drift during COW batch: ${err.message}`;
557
- this.manager.logger.log(`[COW] Recovering from on-chain size drift via recovery sync: ${err.message}`, 'warn');
558
- await this._triggerStateRecoverySync(reason);
559
- return {
560
- executed: false,
561
- hadRotation: false,
562
- recoveredBySync: true,
563
- reason: 'ORDER_SIZE_DRIFT'
564
- };
397
+ return DexbotStateRecovery.recoverBatchSizeDrift(this, err, opContexts);
565
398
  }
566
399
  /**
567
400
  * Extract chain order IDs from opContexts for operations that could
@@ -570,18 +403,7 @@ class DEXBot {
570
403
  * @returns {string[]} Unique chain order IDs
571
404
  */
572
405
  _extractSizeDriftOrderIds(opContexts) {
573
- if (!Array.isArray(opContexts))
574
- return [];
575
- const ids = new Set();
576
- for (const ctx of opContexts) {
577
- if (ctx?.kind === 'size-update' && ctx?.updateInfo?.partialOrder?.orderId) {
578
- ids.add(ctx.updateInfo.partialOrder.orderId);
579
- }
580
- else if (ctx?.kind === 'rotation' && ctx?.rotation?.oldOrder?.orderId) {
581
- ids.add(ctx.rotation.oldOrder.orderId);
582
- }
583
- }
584
- return Array.from(ids);
406
+ return DexbotStateRecovery.extractSizeDriftOrderIds(opContexts);
585
407
  }
586
408
  /**
587
409
  * Reload the entire grid from the persisted on-disk snapshot and reconcile
@@ -594,83 +416,7 @@ class DEXBot {
594
416
  * @returns {Promise<{success: boolean, reason?: string}>}
595
417
  */
596
418
  async _recoverFromPersistedGrid() {
597
- if (!this.accountOrders || !this.manager) {
598
- return { success: false, reason: 'accountOrders or manager unavailable' };
599
- }
600
- const accountRef = this.accountId || this.account?.id || this.account;
601
- if (!accountRef) {
602
- return { success: false, reason: 'no account reference' };
603
- }
604
- this.manager.logger.log('[RECOVERY] Attempting full grid reload from persisted snapshot...', 'warn');
605
- try {
606
- // 1. Force reload from disk
607
- const persistedGrid = this.accountOrders.loadGrid(true);
608
- if (!persistedGrid || persistedGrid.length === 0) {
609
- return { success: false, reason: 'no persisted grid on disk' };
610
- }
611
- const boundaryIdx = this.accountOrders.loadBoundaryIdx(true);
612
- // 2. Load into manager (same path as startup at line 1340)
613
- await Grid.loadGrid(this.manager, persistedGrid, boundaryIdx);
614
- // Gap 1: Grid snapshot sanity check — shared logic with startup path.
615
- if (await this._rejectCorruptedGridSnapshot('recovery')) {
616
- return { success: false, reason: 'corrupted grid snapshot rejected (fund drift)' };
617
- }
618
- // 3. Read current chain state
619
- const chainOpenOrders = await chainOrders.readOpenOrders(accountRef);
620
- // 4. Reconcile
621
- if (chainOpenOrders.length > 0 && this.manager?.syncFromOpenOrders) {
622
- await this.manager.syncFromOpenOrders(chainOpenOrders, {
623
- skipAccounting: true,
624
- });
625
- }
626
- // 5. Persist the reconciled state
627
- if (typeof this.manager.persistGrid === 'function') {
628
- await this.manager.persistGrid();
629
- }
630
- this.manager.logger.log(`[RECOVERY] Grid reloaded from persisted snapshot: ${this.manager.orders.size} orders, ` +
631
- `${chainOpenOrders.length} on-chain orders synced`, 'info');
632
- // Gap 2: Check for unmatched chain orders after reload + sync.
633
- // If the sync resolved all unmatched entries, _lastUnmatchedChainOrders
634
- // was cleared by the sync engine. If any remain, the persisted snapshot
635
- // produced an inconsistent grid — reject so the structural resync
636
- // falls through to requestGridReset (full rebuild from chain).
637
- const remainingUnmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
638
- ? this.manager._lastUnmatchedChainOrders
639
- : [];
640
- if (remainingUnmatched.length > 0) {
641
- const sample = remainingUnmatched.slice(0, 3)
642
- .map(o => this._formatUnmatchedChainOrderForLog(o))
643
- .join(' | ');
644
- this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but ${remainingUnmatched.length} unmatched chain order(s) ` +
645
- `remain${sample ? ` (${sample})` : ''}. Rejecting — full grid reset required.`, 'warn');
646
- return { success: false, reason: `grid inconsistent after reload: ${remainingUnmatched.length} unmatched remain` };
647
- }
648
- // If the reloaded grid is still bloated, reject recovery so the
649
- // caller falls through to a full grid reset (requestGridReset).
650
- // Without this check, a bloated snapshot gets accepted as "success"
651
- // and the structural-resync loop loads the same broken state forever.
652
- //
653
- // NOTE: loadGrid() already fires requestStructuralGridResync when it
654
- // detects bloat internally, so the inner async resync may be in flight
655
- // by the time this outer check runs. That's fine — the structural-resync
656
- // gate (_structuralGridResyncRunning / _structuralGridResyncTimer) dedup's
657
- // concurrent requests. This outer check exists so the synchronous return
658
- // value is honest about the state; the inner resync is a safety net.
659
- const { isGridBloated } = require('./order/grid');
660
- const ordersArr = Array.from(this.manager.orders.values());
661
- const bloatPostRecovery = isGridBloated(this.manager, ordersArr);
662
- if (bloatPostRecovery.bloated) {
663
- const d = bloatPostRecovery.details;
664
- this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but still bloated ` +
665
- `(${d.gridSize} slots, max ${d.maxAllowed}). Rejecting — full grid reset required.`, 'warn');
666
- return { success: false, reason: 'grid still bloated after reload' };
667
- }
668
- return { success: true };
669
- }
670
- catch (err) {
671
- this.manager.logger.log(`[RECOVERY] Full grid reload from persisted snapshot failed: ${err.message}`, 'error');
672
- return { success: false, reason: err.message };
673
- }
419
+ return DexbotStateRecovery.recoverFromPersistedGrid(this);
674
420
  }
675
421
  /**
676
422
  * Reject a corrupted grid snapshot when catastrophic fund drift is detected.
@@ -681,25 +427,7 @@ class DEXBot {
681
427
  * @returns {Promise<boolean>} True if the snapshot was rejected (cleared).
682
428
  */
683
429
  async _rejectCorruptedGridSnapshot(context) {
684
- if (!this.manager?.checkFundDriftAfterFills)
685
- return false;
686
- const driftCheck = this.manager.checkFundDriftAfterFills();
687
- if (driftCheck.isValid)
688
- return false;
689
- const tag = context === 'recovery' ? '[RECOVERY][SNAPSHOT-REJECT]' : '[SNAPSHOT-REJECT]';
690
- this._warn(`${tag} Corrupted grid snapshot detected: ` +
691
- `drift sell=${driftCheck.driftSell.toFixed(2)} buy=${driftCheck.driftBuy.toFixed(2)}. ` +
692
- `Deleting corrupted snapshot.`);
693
- if (this.accountOrders && typeof this.accountOrders.clearGrid === 'function') {
694
- try {
695
- await this.accountOrders.clearGrid();
696
- this._warn(`${tag} Corrupted grid snapshot deleted.`);
697
- }
698
- catch (clearErr) {
699
- this._warn(`${tag} Failed to delete corrupted snapshot: ${clearErr.message}`);
700
- }
701
- }
702
- return true;
430
+ return DexbotStateRecovery.rejectCorruptedGridSnapshot(this, context);
703
431
  }
704
432
  /**
705
433
  * Attempt to repair size-drift for specific order IDs by reading their
@@ -709,141 +437,16 @@ class DEXBot {
709
437
  * @returns {Promise<boolean>} True if all affected orders were repaired
710
438
  */
711
439
  async _targetedOrderRepair(orderIds) {
712
- try {
713
- const objects = await BitShares.db.get_objects(orderIds);
714
- if (!Array.isArray(objects) || objects.length !== orderIds.length)
715
- return false;
716
- const updates = [];
717
- for (let i = 0; i < orderIds.length; i++) {
718
- const chainOrder = objects[i];
719
- const gridOrder = Array.from(this.manager.orders.values())
720
- .find((o) => o.orderId === orderIds[i]);
721
- if (!gridOrder)
722
- continue;
723
- if (!chainOrder || typeof chainOrder.for_sale === 'undefined') {
724
- // Order no longer exists on chain -> fully filled or cancelled.
725
- updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
726
- }
727
- else {
728
- const chainUnits = Number(chainOrder.for_sale);
729
- if (Number.isFinite(chainUnits)) {
730
- const { blockchainToFloat } = require('./order/utils/math');
731
- const prec = gridOrder.type === ORDER_TYPES.SELL
732
- ? this.manager.assets.assetA.precision
733
- : this.manager.assets.assetB.precision;
734
- const floatSize = blockchainToFloat(chainUnits, prec);
735
- if (floatSize !== gridOrder.size) {
736
- updates.push({
737
- id: gridOrder.id,
738
- size: floatSize,
739
- rawOnChain: chainOrder,
740
- });
741
- }
742
- }
743
- }
744
- }
745
- if (updates.length > 0) {
746
- await this._applyRecoverableGridUpdates(updates, 'targeted-size-drift-repair');
747
- }
748
- return true;
749
- }
750
- catch (err) {
751
- this.manager.logger.log(`[COW] Targeted order repair failed: ${err.message}`, 'debug');
752
- return false;
753
- }
440
+ return DexbotStateRecovery.targetedOrderRepair(this, orderIds);
754
441
  }
755
442
  /**
756
443
  * Initialize bot state from storage and blockchain.
757
444
  * Consolidates common initialization logic for start() and startWithPrivateKey().
758
- * @returns {{persistedGrid: Object, persistedBtsFeesOwed: number, persistedBoundaryIdx: number, persistedBtsBalance: number}}
445
+ * @returns {{persistedGrid: Object, persistedBtsFeesOwed: number, persistedBoundaryIdx: number, persistedBtsBalance: number, persistedRecentFillKeys: Object}}
759
446
  * @private
760
447
  */
761
448
  async _initializeStartupState() {
762
- // Create AccountOrders with bot-specific file (one file per bot)
763
- this.accountOrders = new AccountOrders({ botKey: this.config.botKey });
764
- this._processedFillStore.configure({
765
- accountOrders: this.accountOrders
766
- });
767
- // Load persisted processed fills to prevent reprocessing after restart
768
- const loadedPersistedFills = this._processedFillStore.loadPersisted({
769
- minTimestamp: Date.now() - this._fillRecordRetentionMs
770
- });
771
- if (loadedPersistedFills > 0) {
772
- this._log(`Loaded ${loadedPersistedFills} persisted fill records to prevent reprocessing`);
773
- }
774
- // Ensure bot metadata is properly initialized in storage BEFORE any Grid operations
775
- const raw = storage.readFile(PROFILES_BOTS_FILE);
776
- const allBotsConfig = parseJsonWithComments(raw).bots || [];
777
- const myBotConfig = allBotsConfig
778
- .map((b, originalIdx) => b.active !== false ? normalizeBotEntry(b, originalIdx) : null)
779
- .find(b => b && b.botKey === this.config.botKey);
780
- if (myBotConfig) {
781
- await this.accountOrders.syncMeta(myBotConfig);
782
- }
783
- if (!this.manager) {
784
- const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
785
- this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
786
- this.manager.account = this.account;
787
- this.manager.accountId = this.accountId;
788
- this.manager.accountOrders = this.accountOrders;
789
- }
790
- this._wireStructuralGridResyncRequest();
791
- this._wireProcessedFillTracking();
792
- this.manager.startBootstrap();
793
- try {
794
- // Fetch account totals from blockchain at startup to initialize funds
795
- try {
796
- if (this.accountId && this.config.assetA && this.config.assetB) {
797
- await this.manager._initializeAssets();
798
- await this.manager.fetchAccountTotals(this.accountId);
799
- this._log('Fetched blockchain account balances at startup');
800
- }
801
- }
802
- catch (err) {
803
- this._log(`Startup balance fetch FAILED: ${err.message}. Order sizing may be incorrect until next successful sync.`, 'error');
804
- }
805
- // Ensure fee cache is initialized before any fill processing that calls getAssetFees().
806
- try {
807
- await initializeFeeCache([this.config || {}], BitShares);
808
- }
809
- catch (err) {
810
- this._log(`Fee cache initialization FAILED: ${err.message}. Fee calculations will use defaults until cache is refreshed.`, 'error');
811
- }
812
- const persistedGrid = this.accountOrders.loadGrid();
813
- // CRITICAL REPAIR: Strip fake orderIds where orderId === id (e.g. "slot-0")
814
- let repairedGrid = persistedGrid;
815
- if (persistedGrid && persistedGrid.length > 0) {
816
- let repairCount = 0;
817
- repairedGrid = persistedGrid.map(order => {
818
- if (order && order.orderId && order.orderId === order.id) {
819
- repairCount++;
820
- const repairedOrder = { ...order, orderId: '' };
821
- if (repairedOrder.state === ORDER_STATES.ACTIVE || repairedOrder.state === ORDER_STATES.PARTIAL) {
822
- repairedOrder.state = ORDER_STATES.VIRTUAL;
823
- }
824
- return repairedOrder;
825
- }
826
- return order;
827
- });
828
- if (repairCount > 0) {
829
- this._log(`[REPAIR] Stripped ${repairCount} fake orderId(s) from persisted grid to restore rebalancing logic.`);
830
- }
831
- }
832
- const persistedBtsFeesOwed = this.accountOrders.loadBtsFeesOwed();
833
- const persistedBoundaryIdx = this.accountOrders.loadBoundaryIdx();
834
- const persistedBtsBalance = this.accountOrders.loadBtsBalance();
835
- const persistedRecentFillKeys = this.accountOrders.loadRecentFillKeys();
836
- return {
837
- persistedGrid: repairedGrid,
838
- persistedBtsFeesOwed,
839
- persistedBoundaryIdx,
840
- persistedBtsBalance,
841
- persistedRecentFillKeys,
842
- };
843
- }
844
- finally {
845
- this.manager.finishBootstrap();
846
- }
449
+ return DexbotStartupRuntime.initializeStartupState(this);
847
450
  }
848
451
  /**
849
452
  * Wire processed fill tracking into the manager.
@@ -989,446 +592,7 @@ class DEXBot {
989
592
  * @private
990
593
  */
991
594
  async _finishStartupSequence(startupState) {
992
- let { persistedGrid, persistedBtsFeesOwed, persistedBoundaryIdx, persistedBtsBalance, persistedRecentFillKeys, } = startupState;
993
- try {
994
- // CRITICAL: Activate fill listener EARLY - before ANY operations that place orders
995
- // This ensures fills during trigger reset and grid initialization are captured
996
- if (typeof this._fillsUnsubscribe === 'function') {
997
- await this._fillsUnsubscribe().catch(() => { });
998
- }
999
- this._fillsUnsubscribe = await chainOrders.listenForFills(this.account || undefined, this._createFillCallback(chainOrders));
1000
- if (typeof this._fillsUnsubscribe !== 'function') {
1001
- this._warn('Fill listener did not provide an unsubscribe handler. Shutdown cleanup may be incomplete.');
1002
- this._fillsUnsubscribe = null;
1003
- }
1004
- this._log('Fill listener activated (ready to process fills during startup)');
1005
- // Register reconnection callback for safety-net sync after websocket reconnect
1006
- if (!this._reconnectUnregister) {
1007
- this._reconnectUnregister = registerReconnectHook(() => {
1008
- this._log('Blockchain connection re-established; scheduling safety-net sync');
1009
- const runSafetyNetSync = async () => {
1010
- if (this.manager && this.accountId && !this._shuttingDown && !this.config.dryRun) {
1011
- // Cap the entire safety-net sync at TIMING.SAFETY_NET_SYNC_TIMEOUT_MS so it can
1012
- // never hold _fillProcessingLock longer than the
1013
- // 20s shutdown lock timeout. readOpenOrders +
1014
- // synchronizeWithChain + batch + persist can be
1015
- // ~45s worst case without a cap, which would
1016
- // stall shutdown until the 20s timeout fires.
1017
- const safetyNetTimeoutMs = this.config.timing?.SAFETY_NET_SYNC_TIMEOUT_MS;
1018
- let safetyNetTimer;
1019
- const workPromise = this.manager._fillProcessingLock.acquire(async () => {
1020
- if (this._shuttingDown)
1021
- return;
1022
- const chainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
1023
- if (this._shuttingDown)
1024
- return;
1025
- const syncResult = await this.manager.synchronizeWithChain(chainOpenOrders, 'readOpenOrders');
1026
- if (this._shuttingDown)
1027
- return;
1028
- if (syncResult?.filledOrders?.length > 0) {
1029
- this._refreshDynamicWeightDistribution('post-reconnect sync fill');
1030
- this._log(`Post-reconnect sync: ${syncResult.filledOrders.length} grid order(s) found filled.`, 'info');
1031
- await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'post-reconnect sync fill');
1032
- if (this._shuttingDown)
1033
- return;
1034
- }
1035
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
1036
- await this.manager.persistGrid();
1037
- // Cancel any dust created by reconnect fills immediately.
1038
- if (!this._shuttingDown) {
1039
- try {
1040
- const reconnectHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
1041
- await this._cancelDustOrders({
1042
- buy: reconnectHealth.buyDustOrders,
1043
- sell: reconnectHealth.sellDustOrders,
1044
- });
1045
- }
1046
- catch (_dustErr) {
1047
- this._warn(`[RECONNECT] Dust cancel failed: ${_dustErr.message}`);
1048
- }
1049
- }
1050
- });
1051
- try {
1052
- await Promise.race([
1053
- workPromise,
1054
- new Promise((_, reject) => {
1055
- safetyNetTimer = setTimeout(() => reject(new Error(`Safety-net sync exceeded ${safetyNetTimeoutMs}ms cap`)), safetyNetTimeoutMs);
1056
- })
1057
- ]);
1058
- }
1059
- catch (capErr) {
1060
- const fallback = await Promise.race([
1061
- workPromise.then(() => ({ ok: true })),
1062
- new Promise(resolve => setTimeout(() => resolve({ ok: false }), 0))
1063
- ]);
1064
- if (fallback.ok) {
1065
- this._log(`Safety-net sync completed despite timeout — ignoring spurious error.`, 'info');
1066
- }
1067
- else {
1068
- this._warn(`Post-reconnect safety-net sync aborted: ${capErr?.message || capErr}`);
1069
- }
1070
- }
1071
- finally {
1072
- if (safetyNetTimer)
1073
- clearTimeout(safetyNetTimer);
1074
- }
1075
- }
1076
- };
1077
- // The setImmediate callback returns a Promise; we MUST attach
1078
- // a .catch so a rejection here does not propagate to
1079
- // process.on('unhandledRejection') and tear down the bot.
1080
- setImmediate(() => {
1081
- runSafetyNetSync().catch(err => {
1082
- try {
1083
- this._warn('Post-reconnect safety-net sync failed: ' + (err?.message || err));
1084
- }
1085
- catch (_warnErr) {
1086
- // _warn itself inaccessible — last line of defense.
1087
- }
1088
- });
1089
- });
1090
- });
1091
- }
1092
- // CRITICAL: Handle any pending trigger file reset FIRST before any other startup operations
1093
- const hadTriggerReset = await this._handlePendingTriggerReset();
1094
- // CRITICAL: After trigger reset, skip normal startup - grid is already fully initialized
1095
- // The trigger reset already did: grid init, order placement, sync, and persistence
1096
- if (hadTriggerReset) {
1097
- this._log('Trigger reset completed. Skipping normal startup grid initialization.');
1098
- // Post-bootstrap validation and fill processing
1099
- await this.manager._fillProcessingLock.acquire(async () => {
1100
- // STEP 1: Check for fills that occurred during trigger reset
1101
- // These are orders that got filled while Grid.recalculateGrid() was running.
1102
- // The filled slots need new orders placed on them.
1103
- if (this._incomingFillQueue.length > 0) {
1104
- this._log(`[POST-RESET] ${this._incomingFillQueue.length} fill(s) detected during trigger reset. Processing...`);
1105
- // Process fills - this will place new orders on the filled slots
1106
- // Use normal fill processing since bootstrap is complete
1107
- const fills = this._incomingFillQueue.splice(0);
1108
- const processedFillKeys = new Set();
1109
- let requiresOpenOrdersSync = false;
1110
- for (const fill of fills) {
1111
- if (!fill || fill.op?.[0] !== 4)
1112
- continue;
1113
- const fillOp = fill.op[1];
1114
- const gridOrder = this.manager.orders.get(fillOp.order_id) ||
1115
- Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
1116
- if (!gridOrder) {
1117
- // CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
1118
- // This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
1119
- let orphanFillKey = buildFillKey(fill);
1120
- if (!orphanFillKey) {
1121
- orphanFillKey = this._buildOrphanFillFallbackKey(fill);
1122
- }
1123
- if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[POST-RESET]', fillOp.order_id)) {
1124
- continue;
1125
- }
1126
- this._log(`[POST-RESET] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
1127
- const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
1128
- context: 'POST-RESET',
1129
- logger: { log: this._log.bind(this) },
1130
- replayMessage: (op) => `[POST-RESET] Replay detected for orphan fill ${op.order_id}; skipping duplicate credit`
1131
- });
1132
- if (accountingResult.status === 'missing_key') {
1133
- requiresOpenOrdersSync = true;
1134
- }
1135
- continue;
1136
- }
1137
- this._log(`[POST-RESET] Processing fill for ${gridOrder.type} order ${gridOrder.id} at price ${gridOrder.price}`);
1138
- const trackedFillKey = buildFillKey(fill);
1139
- if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[POST-RESET]', fillOp.order_id)) {
1140
- continue;
1141
- }
1142
- this.manager.lockOrders([gridOrder.id]);
1143
- try {
1144
- const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
1145
- context: 'POST-RESET',
1146
- logger: { log: this._log.bind(this) },
1147
- replayMessage: (op) => `[POST-RESET] Replay detected for ${op.order_id}; skipping duplicate rebalance`
1148
- });
1149
- if (accountingResult.status === 'missing_key') {
1150
- requiresOpenOrdersSync = true;
1151
- continue;
1152
- }
1153
- if (accountingResult.status !== 'applied') {
1154
- continue;
1155
- }
1156
- // Process this fill through the full rebalance pipeline
1157
- // This will shift the boundary and place a new order on the filled slot
1158
- const result = await this._processFillsWithBatching([gridOrder], new Set(), `[POST-RESET] fill ${gridOrder.id}`);
1159
- if (result.aborted) {
1160
- this._warn('[POST-RESET] Aborted batch due to illegal state; skipping grid persistence this cycle');
1161
- continue;
1162
- }
1163
- }
1164
- finally {
1165
- this.manager.unlockOrders([gridOrder.id]);
1166
- }
1167
- }
1168
- if (requiresOpenOrdersSync) {
1169
- this._log('[POST-RESET] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
1170
- const postResetChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
1171
- const syncResult = await this.manager.syncFromOpenOrders(postResetChainOpenOrders);
1172
- if (syncResult.filledOrders?.length > 0) {
1173
- await this._processFillsWithBatching(syncResult.filledOrders, new Set(), '[POST-RESET] open-orders fallback');
1174
- }
1175
- }
1176
- await this._flushProcessedFillPersistence('post-reset-batch');
1177
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
1178
- await this.manager.persistGrid();
1179
- }
1180
- // STEP 2: Refresh chain truth before spread correction. Trigger
1181
- // reset can create/cancel orders and fills can arrive while the
1182
- // reset is running; spread decisions must not use stale local grid.
1183
- const { aborted: postResetAborted, hasUnmatched: postResetUnmatched } = await this._syncOpenOrdersAndProcessFills('[POST-RESET] pre-spread');
1184
- if (postResetUnmatched) {
1185
- this._warn(`[POST-RESET] Skipping spread correction: ${postResetUnmatched} unmatched chain order(s) require maintenance reconciliation`);
1186
- }
1187
- // STEP 3: Spread check AFTER fills are processed and chain truth refreshed
1188
- await this.manager.recalculateFunds();
1189
- if (!postResetAborted && !postResetUnmatched) {
1190
- const spreadResult = await this.manager.checkSpreadCondition(BitShares, this.updateOrdersOnChainPlan.bind(this));
1191
- if (spreadResult && spreadResult.ordersPlaced > 0) {
1192
- this._log(`✓ Spread correction after trigger reset: ${spreadResult.ordersPlaced} order(s) placed`);
1193
- await this._persistAndRecoverIfNeeded();
1194
- }
1195
- }
1196
- // Cancel any dust created by post-reset fills immediately.
1197
- if (!this._shuttingDown) {
1198
- try {
1199
- const postResetHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
1200
- await this._cancelDustOrders({
1201
- buy: postResetHealth.buyDustOrders,
1202
- sell: postResetHealth.sellDustOrders,
1203
- });
1204
- }
1205
- catch (_dustErr) {
1206
- this._warn(`[POST-RESET] Dust cancel failed: ${_dustErr.message}`);
1207
- }
1208
- }
1209
- this._log('Bootstrap phase complete - fill processing resumed', 'info');
1210
- });
1211
- await this._setupTriggerFileDetection();
1212
- await this._setupCreditRuntime();
1213
- await this._refreshAndSyncCreditRuntime();
1214
- this._setupBlockchainFetchInterval();
1215
- this._setupCreditWatchdogInterval();
1216
- this._setupCredentialDaemonWatchdogInterval();
1217
- this._setupDustHealthCheckInterval();
1218
- if (this._isOpenOrdersSyncLoopEnabled()) {
1219
- this._startOpenOrdersSyncLoop();
1220
- }
1221
- else {
1222
- this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
1223
- }
1224
- this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
1225
- return; // Skip normal startup path
1226
- }
1227
- // Restore persisted BTS fee
1228
- // SAFE: Done at startup before orders are created, and within fill lock when needed
1229
- this.manager.resetFunds();
1230
- // CRITICAL FIX: Restore BTS fees owed from persistence
1231
- if (persistedBtsFeesOwed && persistedBtsFeesOwed > 0) {
1232
- this.manager.funds.btsFeesOwed = Number(persistedBtsFeesOwed);
1233
- }
1234
- // Restore BTS balance for non-BTS pairs
1235
- if (this.config.assetA !== 'BTS' && this.config.assetB !== 'BTS') {
1236
- if (persistedBtsBalance && typeof persistedBtsBalance === 'object') {
1237
- this.manager.btsBalance = {
1238
- free: persistedBtsBalance.free || 0,
1239
- total: persistedBtsBalance.total || 0,
1240
- locked: persistedBtsBalance.locked || 0
1241
- };
1242
- }
1243
- }
1244
- // Restore recently queued fill keys for crash-durable dedup window.
1245
- // All persisted keys are restored; the in-memory TTL (_fillDedupeWindowMs)
1246
- // will naturally evict any that are already expired on the next fill cycle.
1247
- if (persistedRecentFillKeys && typeof persistedRecentFillKeys === 'object') {
1248
- for (const [fillKey, timestamp] of Object.entries(persistedRecentFillKeys)) {
1249
- this._recentlyQueuedFills.set(fillKey, Number(timestamp));
1250
- }
1251
- this._log(`Restored ${Object.keys(persistedRecentFillKeys).length} recently queued fill key(s) from persisted snapshot`, 'debug');
1252
- }
1253
- if (!this.config.dryRun && !this.accountId) {
1254
- throw new Error('Cannot start bot without a resolved account ID');
1255
- }
1256
- // Use this.accountId which was set during initialize()
1257
- const chainOpenOrders = this.config.dryRun ? [] : await chainOrders.readOpenOrders(this.accountId);
1258
- let shouldRegenerate = false;
1259
- if (!persistedGrid || persistedGrid.length === 0) {
1260
- shouldRegenerate = true;
1261
- this._log('No persisted grid found. Generating new grid.');
1262
- }
1263
- else {
1264
- await this.manager._initializeAssets();
1265
- const decision = await decideStartupGridAction({
1266
- persistedGrid,
1267
- chainOpenOrders,
1268
- manager: this.manager,
1269
- logger: { log: (msg) => this._log(msg) },
1270
- storeGrid: async (orders) => {
1271
- // Pass the snapshot directly to persistGrid so the live
1272
- // `manager.orders` map is not briefly swapped (which would
1273
- // leave the _ordersByState/_ordersByType indexes inconsistent
1274
- // with the map for the duration of the persist call).
1275
- await this.manager.persistGrid(orders);
1276
- },
1277
- attemptResumeFn: attemptResumePersistedGridByPriceMatch,
1278
- });
1279
- shouldRegenerate = decision.shouldRegenerate;
1280
- if (shouldRegenerate && chainOpenOrders.length === 0) {
1281
- this._log('Persisted grid found, but no matching active orders on-chain. Generating new grid.');
1282
- }
1283
- }
1284
- // Restore BTS fees owed ONLY if we're NOT regenerating the grid
1285
- if (!shouldRegenerate) {
1286
- // CRITICAL: Restore BTS fees owed from blockchain operations
1287
- if (persistedBtsFeesOwed > 0) {
1288
- this.manager.funds.btsFeesOwed = persistedBtsFeesOwed;
1289
- this._log(`✓ Restored BTS fees owed: ${Format.formatAmount8(persistedBtsFeesOwed)} BTS`);
1290
- }
1291
- }
1292
- else {
1293
- this._log(`ℹ Grid regenerating - resetting BTS fees to clean state`);
1294
- this.manager.funds.btsFeesOwed = 0;
1295
- }
1296
- // CRITICAL: Use fill lock during ENTIRE startup synchronization to prevent races.
1297
- // This includes grid init, finishBootstrap, and maintenance - all in one atomic block.
1298
- // Lock order: _fillProcessingLock → _divergenceLock (canonical order, same as _consumeFillQueue)
1299
- await this.manager._fillProcessingLock.acquire(async () => {
1300
- try {
1301
- this._refreshDynamicWeightDistribution('startup');
1302
- if (shouldRegenerate) {
1303
- await this.manager._initializeAssets();
1304
- if (Array.isArray(chainOpenOrders) && chainOpenOrders.length > 0) {
1305
- this._log('Generating new grid and syncing with existing on-chain orders...');
1306
- await Grid.initializeGrid(this.manager);
1307
- await this.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
1308
- const rebalanceResult = await reconcileGridOrders({
1309
- manager: this.manager,
1310
- config: this.config,
1311
- account: this.account,
1312
- privateKey: this.privateKey,
1313
- chainOrders,
1314
- chainOpenOrders,
1315
- });
1316
- await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
1317
- }
1318
- else {
1319
- this._log('Generating new grid and placing initial orders on-chain...');
1320
- await this.placeInitialOrders();
1321
- }
1322
- await this._persistAndRecoverIfNeeded();
1323
- }
1324
- else {
1325
- this._log('Found active session. Loading and syncing existing grid.');
1326
- await Grid.loadGrid(this.manager, persistedGrid, persistedBoundaryIdx);
1327
- let startupChainOpenOrders = chainOpenOrders;
1328
- const syncResult = await this.manager.syncFromOpenOrders(startupChainOpenOrders, { skipAccounting: true });
1329
- // Process price corrections queued during startup sync.
1330
- // These are not picked up by _consumeFillQueue until a fill
1331
- // arrives, which may never come for an idle market.
1332
- if (syncResult.ordersNeedingCorrection?.length > 0) {
1333
- await correctAllPriceMismatches(this.manager, this.account, this.privateKey, chainOrders);
1334
- }
1335
- if (syncResult.filledOrders && syncResult.filledOrders.length > 0) {
1336
- this._log(`Startup sync: ${syncResult.filledOrders.length} grid order(s) found filled. Processing proceeds.`, 'info');
1337
- const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'startup sync fill rebalance', { skipAccountTotalsUpdate: true });
1338
- if (!batchResult?.aborted) {
1339
- // Refresh open orders so startup reconcile works with post-batch chain reality
1340
- // and avoids reconciling against a stale pre-batch snapshot.
1341
- startupChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
1342
- await this.manager.synchronizeWithChain(startupChainOpenOrders, 'readOpenOrders');
1343
- }
1344
- }
1345
- const rebalanceResult = await reconcileGridOrders({
1346
- manager: this.manager,
1347
- config: this.config,
1348
- account: this.account,
1349
- privateKey: this.privateKey,
1350
- chainOrders,
1351
- chainOpenOrders: startupChainOpenOrders,
1352
- });
1353
- await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (loaded grid)');
1354
- // Dust state is no longer persisted — cancelled immediately on detection.
1355
- await this._persistAndRecoverIfNeeded();
1356
- // Gap 1: Grid snapshot sanity check — shared logic with recovery path.
1357
- await this._rejectCorruptedGridSnapshot('startup');
1358
- }
1359
- // Drain any fills that arrived during startup while still in bootstrap
1360
- // mode. Safe to call directly since we already hold _fillProcessingLock
1361
- // and _processFillsWithBootstrapMode does NOT re-acquire it.
1362
- if (this._incomingFillQueue.length > 0) {
1363
- this._log(`[STARTUP] Processing ${this._incomingFillQueue.length} queued fill(s) before bootstrap ends`);
1364
- await this._processFillsWithBootstrapMode(chainOrders);
1365
- }
1366
- this.manager.finishBootstrap();
1367
- // Refresh account totals after bootstrap to eliminate the timing
1368
- // gap between the initial balance fetch and grid operations (sync,
1369
- // reconcile, fills). Without this, the first maintenance cycle sees
1370
- // a fund drift (expected at bootstrap) and triggers an unnecessary
1371
- // invariant violation + full recovery cycle.
1372
- // Bound by a timeout to avoid blocking _fillProcessingLock on a
1373
- // flaky node. If the fetch times out, continue with cached values;
1374
- // the next periodic maintenance cycle will retry.
1375
- const FETCH_TIMEOUT_MS = 30000;
1376
- let _fetchTimeoutHandle;
1377
- try {
1378
- await Promise.race([
1379
- this.manager.fetchAccountTotals(),
1380
- new Promise((_, reject) => {
1381
- _fetchTimeoutHandle = setTimeout(() => reject(new Error('timeout')), FETCH_TIMEOUT_MS);
1382
- })
1383
- ]);
1384
- }
1385
- catch (fetchErr) {
1386
- this._log(`[STARTUP] [${this.config?.botKey || 'unknown'}] fetchAccountTotals ${fetchErr.message === 'timeout' ? 'timed out' : 'failed'} (${fetchErr.message}). Continuing with cached account totals.`, 'warn');
1387
- }
1388
- finally {
1389
- clearTimeout(_fetchTimeoutHandle);
1390
- }
1391
- // Perform initial grid maintenance (thresholds, divergence, spread, health)
1392
- // Consolidated into shared logic to ensure consistent behavior at boot and runtime.
1393
- // CRITICAL: Pass lockAlreadyHeld since we're inside _fillProcessingLock.acquire()
1394
- await this._runGridMaintenance('startup');
1395
- // Cancel any dust from a previous bot lifetime immediately.
1396
- const startupHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
1397
- await this._cancelDustOrders({
1398
- buy: startupHealth.buyDustOrders,
1399
- sell: startupHealth.sellDustOrders,
1400
- });
1401
- this._log('Bootstrap phase complete - fill processing resumed', 'info');
1402
- }
1403
- finally {
1404
- // CRITICAL: Always clear bootstrap flag, even on error
1405
- this.manager.finishBootstrap();
1406
- }
1407
- });
1408
- await this._setupTriggerFileDetection();
1409
- await this._setupCreditRuntime();
1410
- await this._refreshAndSyncCreditRuntime();
1411
- await this._runCreditRuntimeMaintenance('startup');
1412
- this._setupBlockchainFetchInterval();
1413
- this._setupCreditWatchdogInterval();
1414
- this._setupCredentialDaemonWatchdogInterval();
1415
- // Periodic dust health check — catches partials that fell below threshold
1416
- // without triggering the post-fill pipeline. Cancels immediately inside the
1417
- // fill-processing lock to avoid racing with fill batches or sync operations.
1418
- this._setupDustHealthCheckInterval();
1419
- if (this._isOpenOrdersSyncLoopEnabled()) {
1420
- this._startOpenOrdersSyncLoop();
1421
- }
1422
- else {
1423
- this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
1424
- }
1425
- this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
1426
- }
1427
- catch (err) {
1428
- this._warn(`Error during grid initialization: ${err.message}`);
1429
- await this.shutdown();
1430
- throw err;
1431
- }
595
+ return DexbotStartupRuntime.finishStartupSequence(this, startupState);
1432
596
  }
1433
597
  /**
1434
598
  * Create the fill callback for listenForFills.
@@ -1447,35 +611,10 @@ class DEXBot {
1447
611
  * @returns {Promise<{syncResult: Object|null, aborted: boolean, hasUnmatched: number, openOrders: Array|null}>}
1448
612
  */
1449
613
  async _syncOpenOrdersAndProcessFills(tag) {
1450
- if (!this.accountId || this.config?.dryRun) {
1451
- return { syncResult: null, aborted: false, hasUnmatched: 0, openOrders: null };
1452
- }
1453
- try {
1454
- let openOrders = await chainOrders.readOpenOrders(this.accountId);
1455
- const syncResult = await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
1456
- let aborted = false;
1457
- if (syncResult?.filledOrders?.length > 0) {
1458
- this._refreshDynamicWeightDistribution(`${tag} sync-fill`);
1459
- this._log(`[SYNC-CHAIN] ${syncResult.filledOrders.length} filled order(s) found during ${tag}`, 'info');
1460
- const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), `${tag} sync-fill`);
1461
- if (!batchResult?.aborted) {
1462
- openOrders = await chainOrders.readOpenOrders(this.accountId);
1463
- await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
1464
- }
1465
- else {
1466
- aborted = true;
1467
- }
1468
- }
1469
- const hasUnmatched = syncResult?.unmatchedChainOrders?.length || 0;
1470
- return { syncResult, aborted, hasUnmatched, openOrders };
1471
- }
1472
- catch (err) {
1473
- this._warn(`[SYNC-CHAIN] Open-orders sync failed during ${tag}: ${err.message}`);
1474
- return { syncResult: null, aborted: true, hasUnmatched: -1, openOrders: null };
1475
- }
614
+ return DexbotMaintenanceRuntime.syncOpenOrdersAndProcessFills(this, tag);
1476
615
  }
1477
616
  _maxConsecutiveFillConsumerFailures() {
1478
- return this.config.fillProcessing?.MAX_CONSECUTIVE_CONSUMER_FAILURES ?? FILL_PROCESSING.MAX_CONSECUTIVE_CONSUMER_FAILURES;
617
+ return DexbotFillRuntime.maxConsecutiveFillConsumerFailures(this);
1479
618
  }
1480
619
  /**
1481
620
  * Compute the backoff delay for fill-consumer retries after the failure
@@ -1487,80 +626,10 @@ class DEXBot {
1487
626
  * @private
1488
627
  */
1489
628
  _computeFillConsumerBackoffMs(failures) {
1490
- const fp = this.config.fillProcessing || FILL_PROCESSING;
1491
- const initial = fp.CONSUMER_BACKOFF_INITIAL_MS;
1492
- const max = fp.CONSUMER_BACKOFF_MAX_MS;
1493
- const stepAfterMax = Math.max(0, failures - this._maxConsecutiveFillConsumerFailures());
1494
- // 0 -> initial, 1 -> 2*initial, 2 -> 4*initial, ... capped at max.
1495
- return Math.min(max, initial * Math.pow(2, stepAfterMax));
629
+ return DexbotFillRuntime.computeFillConsumerBackoffMs(this, failures);
1496
630
  }
1497
631
  _scheduleFillConsumerRestart(chainOrders) {
1498
- const failures = this._consecutiveConsumeFailures;
1499
- if (failures >= this._maxConsecutiveFillConsumerFailures()) {
1500
- // Past the failure budget: switch from tight setImmediate loop to
1501
- // exponential backoff. The consumer continues to retry — a transient
1502
- // outage (e.g., credential daemon recovery) will resume normal
1503
- // operation as soon as one cycle succeeds and resets the counter.
1504
- // A permanent failure mode yields slower-but-still-progressing
1505
- // retries capped at CONSUMER_BACKOFF_MAX_MS, with no permanent
1506
- // stop that would require a bot restart.
1507
- const backoffMs = this._computeFillConsumerBackoffMs(failures);
1508
- const elapsedSec = this._consumeFailureFirstAt
1509
- ? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
1510
- : null;
1511
- const elapsed = elapsedSec !== null ? `${elapsedSec}s` : 'unknown';
1512
- // Escalate log level on sustained failure so operators monitoring
1513
- // for error/critical alerts are not blind to a stuck consumer.
1514
- // - warn : within the first escalation window (5-9 failures,
1515
- // <5 min) — could be a slow recovery
1516
- // - error : 10+ failures OR 5+ minutes of sustained failure
1517
- // - critical: 20+ failures OR 15+ minutes of sustained failure —
1518
- // this is the "permanent fault" signal
1519
- const sustainedLevel = (failures >= 20 || (elapsedSec !== null && elapsedSec >= 900))
1520
- ? 'critical'
1521
- : (failures >= 10 || (elapsedSec !== null && elapsedSec >= 300))
1522
- ? 'error'
1523
- : 'warn';
1524
- this._log(`[FILL-QUEUE] Fill consumer has failed ${failures} consecutive times over ${elapsed}; ` +
1525
- `backing off ${Math.round(backoffMs / TIMING.MILLISECONDS_PER_SECOND)}s before retry. ` +
1526
- `Queue: ${this._incomingFillQueue.length} fills.`, sustainedLevel);
1527
- setTimeout(() => {
1528
- if (this._shuttingDown)
1529
- return;
1530
- this._consumeFillQueue(chainOrders).catch(err => {
1531
- if (!this._consumeFailureFirstAt) {
1532
- this._consumeFailureFirstAt = Date.now();
1533
- }
1534
- this._consecutiveConsumeFailures++;
1535
- const newFailures = this._consecutiveConsumeFailures;
1536
- const newElapsedSec = this._consumeFailureFirstAt
1537
- ? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
1538
- : null;
1539
- const resumeLevel = (newFailures >= 20 || (newElapsedSec !== null && newElapsedSec >= 900))
1540
- ? 'critical'
1541
- : (newFailures >= 10 || (newElapsedSec !== null && newElapsedSec >= 300))
1542
- ? 'error'
1543
- : 'warn';
1544
- this._log(`Fill consumer resume after backoff failed ` +
1545
- `(${newFailures} total, ` +
1546
- `next backoff ${Math.round(this._computeFillConsumerBackoffMs(newFailures) / TIMING.MILLISECONDS_PER_SECOND)}s): ` +
1547
- `${err.message}`, resumeLevel);
1548
- // Continue the backoff loop. The success path of
1549
- // _consumeFillQueue resets the counter, breaking the cycle.
1550
- this._scheduleFillConsumerRestart(chainOrders);
1551
- });
1552
- }, backoffMs);
1553
- return;
1554
- }
1555
- setImmediate(() => this._consumeFillQueue(chainOrders).catch(err => {
1556
- if (!this._consumeFailureFirstAt) {
1557
- this._consumeFailureFirstAt = Date.now();
1558
- }
1559
- this._consecutiveConsumeFailures++;
1560
- const remaining = this._maxConsecutiveFillConsumerFailures() - this._consecutiveConsumeFailures;
1561
- this._log(`Fill consumer failed (${this._consecutiveConsumeFailures}/${this._maxConsecutiveFillConsumerFailures()}, ` +
1562
- `${remaining} attempts remaining): ${err.message}`, this._consecutiveConsumeFailures >= 3 ? 'warn' : 'error');
1563
- }));
632
+ DexbotFillRuntime.scheduleFillConsumerRestart(this, chainOrders);
1564
633
  }
1565
634
  /**
1566
635
  * Consume queued fills from incomingFillQueue and rebalance.
@@ -1576,564 +645,7 @@ class DEXBot {
1576
645
  * @private
1577
646
  */
1578
647
  async _consumeFillQueue(chainOrders) {
1579
- // Helper: every early return below is a "deferral", not a failure.
1580
- // The counter only tracks actual failures, so any healthy deferral
1581
- // path should also reset the counter. Without this, a sequence of
1582
- // F-S-F-S-F-S-F-S-F (fail, succeed, fail, succeed, ...) would still
1583
- // reach the max and trip backoff, OR a failure followed by an empty
1584
- // queue / shutdown / in-flight batch would leave the counter sticky
1585
- // and trigger backoff one step sooner on the next real failure.
1586
- const resetFailureWatchdogIfSet = () => {
1587
- if (this._consecutiveConsumeFailures > 0 || this._consumeFailureFirstAt > 0) {
1588
- this._consecutiveConsumeFailures = 0;
1589
- this._consumeFailureFirstAt = 0;
1590
- }
1591
- };
1592
- // ATOMIC: Only attempt lock acquisition if queue has work
1593
- // This prevents unnecessary lock contention on empty queues
1594
- if (this._incomingFillQueue.length === 0) {
1595
- // Empty queue = consumer is healthy, just idle.
1596
- resetFailureWatchdogIfSet();
1597
- return;
1598
- }
1599
- // Check shutdown state
1600
- if (this._shuttingDown) {
1601
- this._warn('Fill processing skipped: shutdown in progress');
1602
- resetFailureWatchdogIfSet();
1603
- return;
1604
- }
1605
- if (this._batchInFlight || this._recoverySyncInFlight) {
1606
- this.manager?.logger?.log?.(`Fill processing deferred: order pipeline active (${this._incomingFillQueue.length} queued)`, 'debug');
1607
- // A batch is in flight, not a failure. The next iteration will
1608
- // either succeed (and reset the counter on the success path) or
1609
- // fail and increment it. Either way, leaving an old failure
1610
- // count here would double-count.
1611
- resetFailureWatchdogIfSet();
1612
- return;
1613
- }
1614
- let pendingFillKeysForCurrentCycle = new Set();
1615
- try {
1616
- // BOOTSTRAP OPTIMIZATION: During bootstrap, prioritize fill processing over grid-wide checks
1617
- // Process fills immediately with side-only rebalancing (no expensive full grid recalculations)
1618
- if (this.manager.isBootstrapping()) {
1619
- // During bootstrap: skip lock contention checks, process fills directly
1620
- let bootstrapSkipped = false;
1621
- await this.manager._fillProcessingLock.acquire(async () => {
1622
- if (!this.manager.isBootstrapping()) {
1623
- // Bootstrap finished while waiting for the lock — no
1624
- // work to do, but the iteration is still healthy.
1625
- bootstrapSkipped = true;
1626
- return;
1627
- }
1628
- await this._processFillsWithBootstrapMode(chainOrders);
1629
- });
1630
- if (bootstrapSkipped) {
1631
- // Bootstrap-mode lock callback returned without doing
1632
- // work; the .acquire() success path at line ~1941 that
1633
- // would normally reset the counter is not reached in
1634
- // this branch. Reset here so a stale counter from a
1635
- // prior failure doesn't carry over.
1636
- resetFailureWatchdogIfSet();
1637
- }
1638
- return;
1639
- }
1640
- // NORMAL MODE: Non-blocking check if lock already has waiters
1641
- // This prevents unbounded queue growth while still ensuring processing
1642
- // Note: We DO proceed if lock is held but has no waiters - we'll wait our turn
1643
- if (this.manager._fillProcessingLock.getQueueLength() > 0) {
1644
- this._metrics.lockContentionEvents++;
1645
- // Deferral, not a failure. Reset the watchdog so the next
1646
- // call (which may now find an empty queue, or process
1647
- // successfully) doesn't inherit a stale counter.
1648
- resetFailureWatchdogIfSet();
1649
- return;
1650
- }
1651
- await this.manager._fillProcessingLock.acquire(async () => {
1652
- // Reset orphan-fill credit timestamp at the start of each
1653
- // fill cycle. It is re-set when orphan fills are credited,
1654
- // and consumed (set to null) on the next fund-invariant check
1655
- // in accounting.ts, which widens tolerance by 5x while set.
1656
- // Written to this.manager (not this) because accounting.ts
1657
- // reads from the OrderManager reference (mgr).
1658
- // Also cleared by _performStateRecovery after a fresh chain
1659
- // fetch. The timestamp value itself is not compared against a
1660
- // window — it acts as a consume-on-read boolean.
1661
- this.manager._orphanFillsCreditedAt = null;
1662
- while (this._incomingFillQueue.length > 0) {
1663
- const batchStartTime = Date.now();
1664
- // Track max queue depth
1665
- this._metrics.maxQueueDepth = Math.max(this._metrics.maxQueueDepth, this._incomingFillQueue.length);
1666
- // 1. Take snapshot of current work (ATOMIC: splice removes and returns fills atomically)
1667
- const allFills = this._incomingFillQueue.splice(0); // Atomically clear and get all fills
1668
- const validFills = [];
1669
- const processedFillKeys = new Set();
1670
- pendingFillKeysForCurrentCycle = new Set();
1671
- let requiresOpenOrdersSync = false;
1672
- // 2. Filter and Deduplicate (Standard Logic)
1673
- for (const fill of allFills) {
1674
- if (fill && fill.op && fill.op[0] === FILL_PROCESSING.OPERATION_TYPE) {
1675
- const fillOp = fill.op[1];
1676
- // SELF-CANCEL GUARD: only drop malformed, cancel-like
1677
- // artifacts for an order the local process just cancelled.
1678
- // Real fill_order ops carry economic data and must still be
1679
- // accounted even if they arrive shortly after a successful
1680
- // cancel broadcast.
1681
- const hasFillEconomics = fillOp?.pays?.asset_id && fillOp?.pays?.amount != null
1682
- && fillOp?.receives?.asset_id && fillOp?.receives?.amount != null;
1683
- if (chainOrders && typeof chainOrders.wasRecentlyOwnCancelled === 'function'
1684
- && chainOrders.wasRecentlyOwnCancelled(fillOp.order_id)
1685
- && !hasFillEconomics) {
1686
- this.manager.logger.log(`[SELF-CANCEL] Skipping non-economic fill artifact for order ${fillOp.order_id} (just cancelled by this bot)`, 'debug');
1687
- continue;
1688
- }
1689
- // ACCOUNT VALIDATION: Verify the filled order belongs to this bot's account/grid
1690
- // Only process fills for orders we actually manage
1691
- const gridOrder = this.manager.orders.get(fillOp.order_id) ||
1692
- Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
1693
- if (!gridOrder) {
1694
- // Check if this order was already freed by stale-order batch cleanup.
1695
- // When a batch fails due to a stale order reference, the cleanup converts the
1696
- // slot to VIRTUAL/SPREAD, releasing committed funds to chainFree. If we also
1697
- // credit the fill proceeds here, we double-count the capital.
1698
- const staleMarkedAt = this._staleCleanedOrderIds.get(fillOp.order_id);
1699
- if (staleMarkedAt != null) {
1700
- const staleAgeMs = Date.now() - staleMarkedAt;
1701
- if (staleAgeMs <= this._staleCleanupRetentionMs) {
1702
- this.manager.logger.log(`[ORPHAN-FILL] Skipping double-credit for stale-cleaned order ${fillOp.order_id} ` +
1703
- `(funds already freed by batch cleanup, age=${staleAgeMs}ms)`, 'warn');
1704
- continue;
1705
- }
1706
- this._staleCleanedOrderIds.delete(fillOp.order_id);
1707
- }
1708
- // Legitimate orphan fill: order was virtualized during sequential processing
1709
- // but a fill arrived afterward. Credit proceeds to maintain fund tracking.
1710
- let orphanFillKey = buildFillKey(fill);
1711
- if (!orphanFillKey) {
1712
- orphanFillKey = this._buildOrphanFillFallbackKey(fill);
1713
- }
1714
- if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[ORPHAN-FILL]', fillOp.order_id)) {
1715
- continue;
1716
- }
1717
- this.manager.logger.log(`[ORPHAN-FILL] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
1718
- const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
1719
- context: 'ORPHAN-FILL',
1720
- replayMessage: (op) => `[ORPHAN-FILL] Replay detected for ${op.order_id}; skipping duplicate credit`
1721
- });
1722
- if (accountingResult.status === 'missing_key') {
1723
- requiresOpenOrdersSync = true;
1724
- }
1725
- // Record orphan fill credit timestamp for fund invariant
1726
- // tolerance widening. Written to this.manager because
1727
- // accounting.ts reads from the OrderManager (mgr).
1728
- this.manager._orphanFillsCreditedAt = Date.now();
1729
- // Don't add to validFills - we can't do rebalancing without a grid slot
1730
- // But the funds are now credited, preventing fund invariant violation
1731
- continue;
1732
- }
1733
- // Process both maker and taker fills for our grid orders
1734
- // Grid validation ensures we only process fills belonging to our account
1735
- // Taker fills are included because the bot may execute market orders or act as taker
1736
- const roleStr = fillOp.is_maker !== false ? 'maker' : 'taker';
1737
- this.manager.logger.log(`Processing ${roleStr} fill for order ${fillOp.order_id}`, 'debug');
1738
- const fillKey = buildFillKey(fill);
1739
- if (!fillKey) {
1740
- this.manager.logger.log(`[FILL] Missing history id for order ${fillOp.order_id} block ${fill.block_num}; deferring to open-orders sync`, 'warn');
1741
- requiresOpenOrdersSync = true;
1742
- continue;
1743
- }
1744
- if (!this._isNewFillKey(fillKey, processedFillKeys, '[FILL]', fillOp.order_id)) {
1745
- continue;
1746
- }
1747
- validFills.push(fill);
1748
- // Log info
1749
- const paysAmount = fillOp.pays ? fillOp.pays.amount : '?';
1750
- const receivesAmount = fillOp.receives ? fillOp.receives.amount : '?';
1751
- this._log(`\n===== FILL DETECTED =====`);
1752
- this._log(`Order ID: ${fillOp.order_id}`);
1753
- this._log(`Pays: ${paysAmount}, Receives: ${receivesAmount}`);
1754
- this._log(`Block: ${fill.block_num} (History ID: ${fill.id || 'N/A'})`);
1755
- this._log(`=========================\n`);
1756
- }
1757
- }
1758
- // Clean up short-lived queue dedupe cache to prevent memory leak.
1759
- const cleanupTimestamp = Date.now();
1760
- let cleanedCount = 0;
1761
- for (const [key, timestamp] of this._recentlyQueuedFills) {
1762
- if (cleanupTimestamp - timestamp > this._fillDedupeWindowMs) {
1763
- this._recentlyQueuedFills.delete(key);
1764
- cleanedCount++;
1765
- }
1766
- }
1767
- if (cleanedCount > 0) {
1768
- this.manager.logger.log(`Cleaned ${cleanedCount} old queued fill records. Remaining: ${this._recentlyQueuedFills.size}`, 'debug');
1769
- }
1770
- if (validFills.length === 0 && !requiresOpenOrdersSync)
1771
- continue; // Loop back for more
1772
- // 3. Sync and Collect Filled Orders
1773
- let allFilledOrders = [];
1774
- let ordersNeedingCorrection = [];
1775
- const pendingGhostOrders = new Set();
1776
- const fillMode = chainOrders.getFillProcessingMode();
1777
- const processValidFills = async (fillsToSync) => {
1778
- let resolvedOrders = [];
1779
- if (fillMode === 'history') {
1780
- this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (history mode)`, 'info');
1781
- // Batch mode for 2+ fills: acquires _gridLock once, batches drift refetch
1782
- if (fillsToSync.length >= 2) {
1783
- const batchResult = await this.manager.syncFromFillHistoryBatch(fillsToSync, {
1784
- persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
1785
- });
1786
- for (const fill of fillsToSync) {
1787
- const fillKey = buildFillKey({
1788
- orderId: fill?.op?.[1]?.order_id,
1789
- blockNum: fill?.block_num,
1790
- historyId: fill?.id
1791
- });
1792
- if (fillKey)
1793
- pendingFillKeysForCurrentCycle.add(fillKey);
1794
- }
1795
- if (batchResult.filledOrders)
1796
- resolvedOrders.push(...batchResult.filledOrders);
1797
- if (batchResult.requiresOpenOrdersSync)
1798
- requiresOpenOrdersSync = true;
1799
- if (batchResult.ghostOrderIds?.length > 0) {
1800
- for (const id of batchResult.ghostOrderIds) {
1801
- pendingGhostOrders.add(id);
1802
- }
1803
- }
1804
- }
1805
- else {
1806
- // Single fill: use individual per-fill path
1807
- for (const fill of fillsToSync) {
1808
- const resultHistory = await this.manager.syncFromFillHistory(fill, {
1809
- persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
1810
- });
1811
- const fillKey = buildFillKey({
1812
- orderId: fill?.op?.[1]?.order_id,
1813
- blockNum: fill?.block_num,
1814
- historyId: fill?.id
1815
- });
1816
- if (fillKey)
1817
- pendingFillKeysForCurrentCycle.add(fillKey);
1818
- // Dust is handled by post-fill detection below.
1819
- if (resultHistory.filledOrders)
1820
- resolvedOrders.push(...resultHistory.filledOrders);
1821
- if (resultHistory.requiresOpenOrdersSync)
1822
- requiresOpenOrdersSync = true;
1823
- if (resultHistory.ghostOrderId)
1824
- pendingGhostOrders.add(resultHistory.ghostOrderId);
1825
- }
1826
- }
1827
- }
1828
- if (fillMode !== 'history' || requiresOpenOrdersSync) {
1829
- if (fillMode === 'history' && requiresOpenOrdersSync) {
1830
- this.manager.logger.log('Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
1831
- }
1832
- this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (open orders mode)`, 'info');
1833
- const chainOpenOrders = await chainOrders.readOpenOrders(this.account);
1834
- const resultOpenOrders = await this.manager.syncFromOpenOrders(chainOpenOrders);
1835
- // Dust is handled by post-fill detection below.
1836
- if (resultOpenOrders.filledOrders)
1837
- resolvedOrders.push(...resultOpenOrders.filledOrders);
1838
- if (resultOpenOrders.ordersNeedingCorrection)
1839
- ordersNeedingCorrection.push(...resultOpenOrders.ordersNeedingCorrection);
1840
- }
1841
- return resolvedOrders;
1842
- };
1843
- this.manager.pauseFundRecalc();
1844
- try {
1845
- // FIX 1: Block-level fill batching — group valid fills by block
1846
- // and process each block group as a unit. This prevents slot
1847
- // collisions when fills from the same block (original + replacement
1848
- // orders both filled) arrive in different sync batches. Processing
1849
- // all fills from a block together lets the sync engine see every
1850
- // fill for overlapping slots simultaneously.
1851
- const fillsByBlock = new Map();
1852
- const fillsWithoutBlock = [];
1853
- for (const fill of validFills) {
1854
- if (fill.block_num != null) {
1855
- const list = fillsByBlock.get(fill.block_num);
1856
- if (list)
1857
- list.push(fill);
1858
- else
1859
- fillsByBlock.set(fill.block_num, [fill]);
1860
- }
1861
- else {
1862
- fillsWithoutBlock.push(fill);
1863
- }
1864
- }
1865
- // Process block groups in ascending block order so the sync
1866
- // engine sees a deterministic, chronological fill sequence.
1867
- const sortedBlocks = [...fillsByBlock.keys()].sort((a, b) => a - b);
1868
- const accumulatedOrders = [];
1869
- let anyRequiresSync = false;
1870
- // Capture whether any fill was skipped during validation
1871
- // (e.g. missing history ID) and set the sync flag. The
1872
- // per-block reset below would lose this state since those
1873
- // fills were filtered out of validFills and never enter
1874
- // any block group.
1875
- const initialRequiresSync = requiresOpenOrdersSync;
1876
- for (const blockNum of sortedBlocks) {
1877
- // Reset requiresOpenOrdersSync per block group so one
1878
- // block's history-id gap doesn't force the next block
1879
- // into an unnecessary open-orders snapshot re-fetch.
1880
- requiresOpenOrdersSync = false;
1881
- this.manager.logger.log(`[FILL-BLOCK] Processing ${fillsByBlock.get(blockNum).length} fill(s) from block ${blockNum}`, 'debug');
1882
- const blockResult = await processValidFills(fillsByBlock.get(blockNum));
1883
- accumulatedOrders.push(...blockResult);
1884
- if (requiresOpenOrdersSync)
1885
- anyRequiresSync = true;
1886
- }
1887
- // Preserve both per-block sync flags and the initial flag
1888
- // from filtered-out fills (missing history ID).
1889
- requiresOpenOrdersSync = anyRequiresSync || initialRequiresSync;
1890
- if (fillsWithoutBlock.length > 0) {
1891
- this.manager.logger.log(`[FILL-BLOCK] Processing ${fillsWithoutBlock.length} fill(s) without block info`, 'debug');
1892
- const noBlockResult = await processValidFills(fillsWithoutBlock);
1893
- accumulatedOrders.push(...noBlockResult);
1894
- if (requiresOpenOrdersSync)
1895
- anyRequiresSync = true;
1896
- }
1897
- // If a fill was filtered out during validation (e.g. missing
1898
- // history ID) and no block group triggered the open-orders
1899
- // sync, run it now so the grid state is reconciled.
1900
- if (requiresOpenOrdersSync && !anyRequiresSync) {
1901
- this.manager.logger.log('[FILL-BLOCK] Running open-orders sync for fills with missing history identifiers', 'warn');
1902
- const fallbackOrders = await processValidFills([]);
1903
- accumulatedOrders.push(...fallbackOrders);
1904
- // Flag consumed — variable goes out of scope on
1905
- // next while iteration.
1906
- }
1907
- allFilledOrders = accumulatedOrders;
1908
- // 4. Handle Price Corrections
1909
- if (ordersNeedingCorrection.length > 0) {
1910
- const correctionResult = await correctAllPriceMismatches(this.manager, this.account, this.privateKey, chainOrders);
1911
- if (correctionResult.failed > 0)
1912
- this.manager.logger.log(`${correctionResult.failed} corrections failed`, 'error');
1913
- }
1914
- }
1915
- finally {
1916
- // 4b. Cancel orphaned chain orders detected by other-side rounding.
1917
- // These orders have a tiny residual that the bot treated as fully
1918
- // filled, but the blockchain did not close the order. We send a
1919
- // cancel tx to clean up the zombie order on chain.
1920
- // Runs in the finally so it still fires even if processValidFills
1921
- // throws after collecting ghost IDs into pendingGhostOrders.
1922
- //
1923
- // OPTIMIZATION: Batch all new ghost cancels into one executeBatch
1924
- // call with multiple cancelOrder ops, reducing per-order tx fees
1925
- // and broadcast overhead. Falls back to individual cancels on
1926
- // batch failure.
1927
- if (pendingGhostOrders.size > 0) {
1928
- if (!this._ghostOrderCancelAttempted)
1929
- this._ghostOrderCancelAttempted = new Set();
1930
- const newGhostIds = [...pendingGhostOrders].filter(id => !this._ghostOrderCancelAttempted.has(id));
1931
- if (newGhostIds.length > 0) {
1932
- const MAX_OPS_PER_TX = 200;
1933
- let batchFailed = false;
1934
- // Build cancel ops with per-ID error tolerance.
1935
- // Index correlates 1:1 with newGhostIds (allSettled preserves order).
1936
- const buildResults = await Promise.allSettled(newGhostIds.map(id => chainOrders.buildCancelOrderOp(this.account, id)));
1937
- const cancelOps = [];
1938
- for (let i = 0; i < buildResults.length; i++) {
1939
- const result = buildResults[i];
1940
- const id = newGhostIds[i];
1941
- if (result.status === 'fulfilled') {
1942
- cancelOps.push(result.value);
1943
- }
1944
- else {
1945
- this.manager.logger.log(`[SYNC] Failed to build cancel op for ghost order ${id}: ${result.reason?.message || result.reason}`, 'warn');
1946
- }
1947
- }
1948
- // Chunk into batches respecting MAX_OPS_PER_TX
1949
- for (let i = 0; i < cancelOps.length; i += MAX_OPS_PER_TX) {
1950
- const chunk = cancelOps.slice(i, i + MAX_OPS_PER_TX);
1951
- const batchIds = newGhostIds.slice(i, i + MAX_OPS_PER_TX);
1952
- try {
1953
- this.manager.logger.log(`[SYNC] Batch-cancelling ${chunk.length} orphaned chain order(s) ` +
1954
- `(batch ${Math.floor(i / MAX_OPS_PER_TX) + 1}/${Math.ceil(cancelOps.length / MAX_OPS_PER_TX)})`, 'info');
1955
- await chainOrders.executeBatch(this.account, this.privateKey, chunk);
1956
- // Mark successfully cancelled IDs immediately
1957
- for (const id of batchIds) {
1958
- this._ghostOrderCancelAttempted.add(id);
1959
- }
1960
- }
1961
- catch (batchErr) {
1962
- batchFailed = true;
1963
- this.manager.logger.log(`[SYNC] Batch ghost cancel failed for chunk ${Math.floor(i / MAX_OPS_PER_TX) + 1} ` +
1964
- `(${chunk.length} orders): ${batchErr?.message || batchErr}`, 'warn');
1965
- }
1966
- }
1967
- // Log overall success if all chunks and all builds succeeded
1968
- if (!batchFailed && cancelOps.length === newGhostIds.length) {
1969
- this.manager.logger.log(`[SYNC] Successfully batch-cancelled ${newGhostIds.length} orphaned chain order(s)`, 'info');
1970
- }
1971
- // Fallback: individual cancels for IDs that were not marked successful
1972
- for (const ghostOrderId of newGhostIds) {
1973
- if (this._ghostOrderCancelAttempted.has(ghostOrderId))
1974
- continue;
1975
- try {
1976
- this.manager.logger.log(`[SYNC] Cancelling orphaned chain order ${ghostOrderId} (other-side full-fill residual)`, 'info');
1977
- await chainOrders.cancelOrder(this.account, this.privateKey, ghostOrderId);
1978
- this._ghostOrderCancelAttempted.add(ghostOrderId);
1979
- }
1980
- catch (err) {
1981
- this.manager.logger.log(`[SYNC] Failed to cancel orphaned order ${ghostOrderId}: ${err?.message || err}`, 'warn');
1982
- }
1983
- }
1984
- }
1985
- }
1986
- await this.manager.resumeFundRecalc();
1987
- }
1988
- // Refresh dynamic weight distribution before processing fills
1989
- // so the rebalance uses the latest market adapter weights, not
1990
- // stale values from the last periodic refresh cycle.
1991
- // Lightweight: reads local JSON snapshot file.
1992
- this._refreshDynamicWeightDistribution('fill queue');
1993
- // 5. Fixed-Cap Fill Rebalance
1994
- // - 1..MAX_FILL_BATCH_SIZE fills: unified full-set planning
1995
- // - larger bursts: fixed-size chunking at MAX_FILL_BATCH_SIZE
1996
- if (allFilledOrders.length > 0) {
1997
- const result = await this._processFillsWithBatching(allFilledOrders, null, 'fill set');
1998
- let abortedFillCycle = result.aborted;
1999
- if (!abortedFillCycle) {
2000
- const batchFillKeys = new Set(allFilledOrders.map(filledOrder => buildFillKey({
2001
- orderId: filledOrder?.orderId,
2002
- blockNum: filledOrder?.blockNum,
2003
- historyId: filledOrder?.historyId
2004
- })).filter(Boolean));
2005
- await this._flushProcessedFillPersistenceForKeys(batchFillKeys, 'fill-batch-committed');
2006
- }
2007
- else {
2008
- this.manager.logger.log('[FILL-DEDUP] Fill cycle aborted; fill key persistence guarded under abort path.', 'warn');
2009
- }
2010
- // 6. Rebalance Recovery Loop (Sequential Extensions)
2011
- // DISABLED FOR SEQUENTIAL: Each sequential fill already triggers a full rebalance with proper
2012
- // boundary shift. An additional recovery loop with EMPTY fills causes the boundary to remain
2013
- // at the last fill's position, leading to wrong operation types (updates instead of rotations)
2014
- // and operations on the wrong side.
2015
- //
2016
- // In the future, recovery loop can be re-enabled for single fills if needed, but ONLY
2017
- // if it passes the actual fills to processFilledOrders so the boundary shifts correctly.
2018
- // For now: Each fill = full rebalance with boundary shift = complete correction in one pass.
2019
- // CRITICAL: Do NOT run spread correction here during sequential fill processing.
2020
- // The rebalance from each fill should maintain spread naturally. Running spread correction
2021
- // immediately after creates new orders that may get filled by market before next cycle,
2022
- // causing cascading fills and potentially SPREAD slots becoming PARTIAL (error condition).
2023
- // Spread correction runs in the main loop instead.
2024
- const fullFillCount = allFilledOrders.filter(o => o && o.isPartial !== true).length;
2025
- const hasAnyFills = allFilledOrders.some(o => o);
2026
- const shouldRunPostFillChecks = !abortedFillCycle && fullFillCount > 0;
2027
- const shouldRunDustDetection = !abortedFillCycle && hasAnyFills;
2028
- if (shouldRunDustDetection) {
2029
- const healthResult = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
2030
- const allDust = [
2031
- ...(healthResult.buyDustOrders || []),
2032
- ...(healthResult.sellDustOrders || []),
2033
- ];
2034
- if (allDust.length > 0) {
2035
- const dustCancelResult = await this._cancelDustOrders({
2036
- buy: healthResult.buyDustOrders,
2037
- sell: healthResult.sellDustOrders,
2038
- });
2039
- if (dustCancelResult?.batchResult?.aborted) {
2040
- abortedFillCycle = true;
2041
- }
2042
- }
2043
- }
2044
- // Run grid maintenance after fills to rebuild degraded grid.
2045
- // CRITICAL FIX (commit a946c33): Replaced inline divergence checks with centralized
2046
- // _runGridMaintenance call to ensure pipeline protection applies consistently.
2047
- // Before: Divergence checks ran immediately after fills, causing race-to-resize
2048
- // After: Grid maintenance waits for isPipelineEmpty() before structural changes
2049
- // Run only when the cycle contains at least one full fill.
2050
- if (shouldRunPostFillChecks && !abortedFillCycle) {
2051
- await this._runGridMaintenance('post-fill');
2052
- }
2053
- }
2054
- else if (pendingFillKeysForCurrentCycle.size > 0) {
2055
- await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, 'fill-batch-no-rotations');
2056
- }
2057
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
2058
- await retryPersistenceIfNeeded(this.manager);
2059
- // Periodically clean up old fill records after processing N fills.
2060
- // Counter is protected by _fillProcessingLock during fill consumption.
2061
- this._fillCleanupCounter += validFills.length;
2062
- const cleanupThreshold = MAINTENANCE.CLEANUP_PROBABILITY > 0 && MAINTENANCE.CLEANUP_PROBABILITY < 1
2063
- ? Math.floor(1 / MAINTENANCE.CLEANUP_PROBABILITY)
2064
- : 100; // Default: every 100 fills
2065
- if (this._fillCleanupCounter >= cleanupThreshold) {
2066
- try {
2067
- await this.accountOrders.cleanOldProcessedFills(TIMING.FILL_RECORD_RETENTION_MS);
2068
- this._fillCleanupCounter = 0; // Reset counter after cleanup (success or retry on next batch if failed)
2069
- }
2070
- catch (err) {
2071
- this.manager?.logger?.log(`Warning: Fill cleanup failed (will retry): ${err.message}`, 'warn');
2072
- }
2073
- }
2074
- // Update metrics
2075
- this._metrics.fillsProcessed += validFills.length;
2076
- this._metrics.fillProcessingTimeMs += Date.now() - batchStartTime;
2077
- // Prune expired stale-cleaned order IDs after each processing cycle.
2078
- if (this._staleCleanedOrderIds.size > 0) {
2079
- const now = Date.now();
2080
- let prunedCount = 0;
2081
- for (const [orderId, markedAt] of this._staleCleanedOrderIds) {
2082
- if (now - markedAt > this._staleCleanupRetentionMs) {
2083
- this._staleCleanedOrderIds.delete(orderId);
2084
- prunedCount++;
2085
- }
2086
- }
2087
- if (prunedCount > 0) {
2088
- this.manager.logger.log(`[STALE-CLEANUP] Pruned ${prunedCount} expired stale-cleaned order IDs ` +
2089
- `(retention=${this._staleCleanupRetentionMs}ms, remaining=${this._staleCleanedOrderIds.size})`, 'debug');
2090
- }
2091
- }
2092
- } // End while(_incomingFillQueue)
2093
- this._markGridActivity('fill processing end');
2094
- // Reset the fill-consumer watchdog on the success path. The
2095
- // counter is only incremented by _scheduleFillConsumerRestart's
2096
- // catch handler; without this reset, a fill pattern of
2097
- // F-S-F-S-F-S-F-S-F would still reach the max and the consumer
2098
- // would stop being re-scheduled.
2099
- this._consecutiveConsumeFailures = 0;
2100
- this._consumeFailureFirstAt = 0;
2101
- });
2102
- }
2103
- catch (err) {
2104
- const isCredentialOutage = this._isCredentialDaemonError(err);
2105
- if (pendingFillKeysForCurrentCycle.size > 0) {
2106
- const flushReason = isCredentialOutage
2107
- ? 'credential-outage-verified-fills'
2108
- : 'fill-cycle-error-verified-fills';
2109
- if (isCredentialOutage) {
2110
- this._credentialRecoveryNeeded = true;
2111
- this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
2112
- }
2113
- try {
2114
- await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, flushReason, { throwOnError: true });
2115
- const credentialSuffix = isCredentialOutage
2116
- ? '; grid persistence is suspended until recovery'
2117
- : '';
2118
- this.manager?.logger?.log?.(`[FILL-DEDUP] Persisted ${pendingFillKeysForCurrentCycle.size} verified processed-fill write(s) after fill cycle error${credentialSuffix}.`, isCredentialOutage ? 'warn' : 'info');
2119
- }
2120
- catch (flushErr) {
2121
- this.manager?.logger?.log?.(`[FILL-DEDUP] Failed to persist verified fill keys during fill error handling: ${flushErr.message}`, 'warn');
2122
- }
2123
- }
2124
- if (isCredentialOutage && pendingFillKeysForCurrentCycle.size === 0) {
2125
- this._credentialRecoveryNeeded = true;
2126
- this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
2127
- }
2128
- this._log(`Error processing fills: ${err.message}`, 'error');
2129
- if (err.stack)
2130
- this._log(err.stack, 'error');
2131
- }
2132
- // Post-processing: If new fills arrived while processing, schedule another cycle
2133
- // SAFE: Done outside lock context, no async work in finally block
2134
- if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
2135
- this._scheduleFillConsumerRestart(chainOrders);
2136
- }
648
+ return DexbotFillRuntime.consumeFillQueue(this, chainOrders);
2137
649
  }
2138
650
  /**
2139
651
  * Process fills during bootstrap phase using the standard fill pipeline.
@@ -2153,98 +665,7 @@ class DEXBot {
2153
665
  * @returns {Promise<void>}
2154
666
  */
2155
667
  async _processFillsWithBootstrapMode(chainOrders) {
2156
- if (this._incomingFillQueue.length === 0)
2157
- return;
2158
- const startTime = Date.now();
2159
- const fills = this._incomingFillQueue.splice(0);
2160
- const validFills = [];
2161
- const processedFillKeys = new Set();
2162
- let requiresOpenOrdersSync = false;
2163
- // 1. Validate and deduplicate fills
2164
- for (const fill of fills) {
2165
- if (!fill || fill.op?.[0] !== 4)
2166
- continue;
2167
- const fillOp = fill.op[1];
2168
- const gridOrder = this.manager.orders.get(fillOp.order_id) ||
2169
- Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
2170
- if (!gridOrder) {
2171
- // CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
2172
- // This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
2173
- let orphanFillKey = buildFillKey(fill);
2174
- if (!orphanFillKey) {
2175
- orphanFillKey = this._buildOrphanFillFallbackKey(fill);
2176
- }
2177
- if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
2178
- continue;
2179
- }
2180
- this.manager.logger.log(`[BOOTSTRAP] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
2181
- const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
2182
- context: 'BOOTSTRAP'
2183
- });
2184
- if (accountingResult.status === 'missing_key') {
2185
- requiresOpenOrdersSync = true;
2186
- }
2187
- continue;
2188
- }
2189
- const trackedFillKey = buildFillKey(fill);
2190
- if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
2191
- continue;
2192
- }
2193
- this.manager.lockOrders([gridOrder.id]);
2194
- try {
2195
- const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
2196
- context: 'BOOTSTRAP',
2197
- replayMessage: (op) => `[BOOTSTRAP] Replay detected for ${op.order_id}; skipping duplicate bootstrap rebalance`
2198
- });
2199
- if (accountingResult.status === 'missing_key') {
2200
- requiresOpenOrdersSync = true;
2201
- continue;
2202
- }
2203
- if (accountingResult.status !== 'applied') {
2204
- continue;
2205
- }
2206
- validFills.push({ ...fill, gridOrder });
2207
- const fillType = gridOrder.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL';
2208
- this._log(`[BOOTSTRAP] Fill detected: ${fillType} order (${fillOp.is_maker !== false ? 'maker' : 'taker'})`);
2209
- }
2210
- finally {
2211
- this.manager.unlockOrders([gridOrder.id]);
2212
- }
2213
- }
2214
- if (requiresOpenOrdersSync) {
2215
- this._log('[BOOTSTRAP] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
2216
- const bootstrapChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
2217
- const syncResult = await this.manager.syncFromOpenOrders(bootstrapChainOpenOrders);
2218
- if (syncResult.filledOrders?.length > 0) {
2219
- const queuedOrderIds = new Set(validFills.map(fill => fill?.gridOrder?.orderId).filter(Boolean));
2220
- for (const filledOrder of syncResult.filledOrders) {
2221
- if (!filledOrder?.orderId || queuedOrderIds.has(filledOrder.orderId))
2222
- continue;
2223
- validFills.push({ gridOrder: filledOrder });
2224
- queuedOrderIds.add(filledOrder.orderId);
2225
- }
2226
- }
2227
- }
2228
- await this._flushProcessedFillPersistence('bootstrap-batch');
2229
- if (validFills.length === 0)
2230
- return;
2231
- // 2. Process fills through the standard fill pipeline.
2232
- // This handles same-side replacement, symmetric rebalance, and dust cleanup
2233
- // via the same path used by the post-reset flow.
2234
- try {
2235
- this._log(`[BOOTSTRAP] Processing ${validFills.length} fill(s) through standard pipeline`, 'info');
2236
- const filledOrders = validFills.map(f => f.gridOrder);
2237
- const result = await this._processFillsWithBatching(filledOrders, new Set(), '[BOOTSTRAP] fill processing');
2238
- if (result.aborted) {
2239
- this._warn('[BOOTSTRAP] Aborted batch due to illegal state; skipping grid persistence this cycle');
2240
- }
2241
- this._metrics.fillsProcessed += validFills.length;
2242
- this._metrics.fillProcessingTimeMs += Date.now() - startTime;
2243
- }
2244
- catch (err) {
2245
- this._warn(`[BOOTSTRAP] Error processing fills: ${err.message}`);
2246
- this.manager.logger.log(`[BOOTSTRAP] Fill error: ${err.message}`, 'error');
2247
- }
668
+ return DexbotFillRuntime.processFillsWithBootstrapMode(this, chainOrders);
2248
669
  }
2249
670
  /**
2250
671
  * Set up account identifier and configure global context.
@@ -2314,43 +735,7 @@ class DEXBot {
2314
735
  * @returns {Promise<void>}
2315
736
  */
2316
737
  async placeInitialOrders() {
2317
- if (!this.manager) {
2318
- const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
2319
- this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
2320
- this.manager.accountOrders = this.accountOrders;
2321
- }
2322
- this._wireStructuralGridResyncRequest();
2323
- this.manager.startBootstrap();
2324
- try {
2325
- try {
2326
- const botFunds = this.config && this.config.botFunds ? this.config.botFunds : {};
2327
- const needsPercent = (v) => typeof v === 'string' && v.includes('%');
2328
- if ((needsPercent(botFunds.buy) || needsPercent(botFunds.sell)) && (this.accountId || this.account)) {
2329
- if (typeof this.manager._fetchAccountBalancesAndSetTotals === 'function') {
2330
- await this.manager._fetchAccountBalancesAndSetTotals();
2331
- }
2332
- }
2333
- }
2334
- catch (errFetch) {
2335
- this._warn(`Could not fetch account totals before initializing grid: ${errFetch && errFetch.message ? errFetch.message : errFetch}`);
2336
- }
2337
- await Grid.initializeGrid(this.manager);
2338
- if (this.config.dryRun) {
2339
- this.manager.logger.log('Dry run enabled, skipping on-chain order placement.', 'info');
2340
- await this.manager.persistGrid();
2341
- return;
2342
- }
2343
- this.manager.logger.log('Placing initial orders on-chain...', 'info');
2344
- const ordersToActivate = this.manager.getInitialOrdersToActivate();
2345
- const orderGroups = this._buildOutsideInPairGroupsForOrders(ordersToActivate);
2346
- for (const group of orderGroups) {
2347
- await this.updateOrdersOnChainPlan({ ordersToPlace: group });
2348
- }
2349
- await this.manager.persistGrid();
2350
- }
2351
- finally {
2352
- this.manager.finishBootstrap();
2353
- }
738
+ return DexbotStartupRuntime.placeInitialOrdersImpl(this);
2354
739
  }
2355
740
  /**
2356
741
  * Build outside-in pair groups for initial order placement.
@@ -2358,11 +743,7 @@ class DEXBot {
2358
743
  * @returns {Array<Array<Object>>} Grouped order arrays
2359
744
  */
2360
745
  _buildOutsideInPairGroupsForOrders(orders) {
2361
- return buildOutsideInPairGroups(orders, {
2362
- isValid: Boolean,
2363
- getType: o => o.type,
2364
- getPrice: o => o.price,
2365
- });
746
+ return cowRuntime.buildOutsideInPairGroupsForOrders(orders);
2366
747
  }
2367
748
  /**
2368
749
  * Build outside-in pair groups for create entry contexts.
@@ -2370,11 +751,7 @@ class DEXBot {
2370
751
  * @returns {Array<Array<Object>>} Grouped entry arrays
2371
752
  */
2372
753
  _buildOutsideInPairGroupsForCreateEntries(createEntries) {
2373
- return buildOutsideInPairGroups(createEntries, {
2374
- isValid: e => Boolean(e?.context?.order),
2375
- getType: e => e.context.order.type,
2376
- getPrice: e => e.context.order.price,
2377
- });
754
+ return cowRuntime.buildOutsideInPairGroupsForCreateEntries(createEntries);
2378
755
  }
2379
756
  /**
2380
757
  * Resolve the centralized fill batch cap.
@@ -2390,19 +767,7 @@ class DEXBot {
2390
767
  * @returns {Array} Array of operation result entries
2391
768
  */
2392
769
  _extractOperationResults(result, warnContext = '') {
2393
- const extracted = extractBatchOperationResults(result);
2394
- if (Array.isArray(extracted))
2395
- return extracted;
2396
- if (result) {
2397
- const resultType = Array.isArray(result) ? 'array' : typeof result;
2398
- const keySummary = (resultType === 'object' && !Array.isArray(result))
2399
- ? Object.keys(result).slice(0, 8).join(',')
2400
- : '';
2401
- const contextSuffix = warnContext ? ` (${warnContext})` : '';
2402
- const keysSuffix = keySummary ? `; keys=[${keySummary}]` : '';
2403
- this.manager?.logger?.log(`[COW] Unrecognized operation_results shape${contextSuffix}; defaulting to empty results. resultType=${resultType}${keysSuffix}`, 'warn');
2404
- }
2405
- return [];
770
+ return cowRuntime.extractOperationResults(result, warnContext, this.manager?.logger?.log?.bind(this.manager?.logger));
2406
771
  }
2407
772
  /**
2408
773
  * Find CREATE operation contexts whose broadcast result did not include a chain order id.
@@ -2412,19 +777,7 @@ class DEXBot {
2412
777
  * @returns {Array<{index:number, ctx:Object}>} Missing create result contexts.
2413
778
  */
2414
779
  _findMissingCreateResultContexts(operationResults, opContexts) {
2415
- const missing = [];
2416
- if (!Array.isArray(opContexts))
2417
- return missing;
2418
- for (let i = 0; i < opContexts.length; i++) {
2419
- const ctx = opContexts[i];
2420
- if (ctx?.kind !== 'create')
2421
- continue;
2422
- const chainOrderId = operationResults?.[i]?.[1];
2423
- if (!chainOrderId || !/^1\.7\.\d+$/.test(String(chainOrderId))) {
2424
- missing.push({ index: i, ctx });
2425
- }
2426
- }
2427
- return missing;
780
+ return cowRuntime.findMissingCreateResultContexts(operationResults, opContexts);
2428
781
  }
2429
782
  /**
2430
783
  * Run an immediate chain sync after a successful CREATE broadcast returned incomplete ids.
@@ -2438,42 +791,7 @@ class DEXBot {
2438
791
  * @returns {Promise<void>}
2439
792
  */
2440
793
  async _recoverAfterMissingCreateResults(reason = 'missing create operation results') {
2441
- try {
2442
- const accountRef = this.accountId || this.account?.id || this.account;
2443
- if (!accountRef || !this.manager || !chainOrders?.readOpenOrders) {
2444
- this.manager?.logger?.log?.(`[COW] Recovery sync unavailable after ${reason}`, 'warn');
2445
- return;
2446
- }
2447
- const preRecoveryMissingCreateBlockers = Array.isArray(this.manager._lastUnmatchedChainOrders)
2448
- ? this.manager._lastUnmatchedChainOrders
2449
- .filter(order => order?.reason === 'missing-create-result')
2450
- .map(order => ({ ...order }))
2451
- : [];
2452
- const openOrders = await chainOrders.readOpenOrders(accountRef);
2453
- const recoveryResult = await this.manager.syncFromOpenOrders(openOrders, {
2454
- skipAccounting: false,
2455
- });
2456
- this._preserveMissingCreateBlockersAfterRecovery(preRecoveryMissingCreateBlockers, recoveryResult);
2457
- // Persist any master grid mutations from the recovery sync. The
2458
- // caller returned from the COW catch handler before reaching the
2459
- // success-path persistGrid.
2460
- if (typeof this.manager.persistGrid === 'function') {
2461
- await this.manager.persistGrid();
2462
- }
2463
- }
2464
- catch (err) {
2465
- this.manager?.logger?.log?.(`[COW] CRITICAL: Recovery sync failed after ${reason}: ${err.message}`, 'error');
2466
- if (typeof this.manager?.requestStructuralGridResync === 'function') {
2467
- try {
2468
- await this.manager.requestStructuralGridResync(`recovery sync failed after ${reason}`, {
2469
- error: err.message
2470
- });
2471
- }
2472
- catch (scheduleErr) {
2473
- this.manager?.logger?.log?.(`[COW] CRITICAL: Failed to schedule structural resync after recovery failure: ${scheduleErr.message}`, 'error');
2474
- }
2475
- }
2476
- }
794
+ return cowRuntime.recoverAfterMissingCreateResults(this, reason);
2477
795
  }
2478
796
  /**
2479
797
  * Restore unresolved missing-create blockers after recovery if sync did not adopt them.
@@ -2483,30 +801,7 @@ class DEXBot {
2483
801
  * @returns {void}
2484
802
  */
2485
803
  _preserveMissingCreateBlockersAfterRecovery(blockers, recoveryResult) {
2486
- if (!Array.isArray(blockers) || blockers.length === 0 || !this.manager)
2487
- return;
2488
- const adoptedSlotIds = new Set((Array.isArray(recoveryResult?.updatedOrders) ? recoveryResult.updatedOrders : [])
2489
- .filter(order => order?.id && order?.orderId)
2490
- .map(order => order.id));
2491
- const unresolvedBlockers = blockers.filter(blocker => !blocker.slotId || !adoptedSlotIds.has(blocker.slotId));
2492
- if (unresolvedBlockers.length === 0)
2493
- return;
2494
- const currentUnmatched = Array.isArray(this.manager._lastUnmatchedChainOrders)
2495
- ? this.manager._lastUnmatchedChainOrders
2496
- : [];
2497
- const currentKeys = new Set(currentUnmatched.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
2498
- const restored = [...currentUnmatched];
2499
- for (const blocker of unresolvedBlockers) {
2500
- const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
2501
- if (!currentKeys.has(key))
2502
- restored.push({ ...blocker });
2503
- }
2504
- if (restored.length !== currentUnmatched.length) {
2505
- this.manager._lastUnmatchedChainOrders = restored;
2506
- this.manager._lastUnmatchedChainOrdersAt = Date.now();
2507
- this.manager.logger?.log?.(`[COW] Preserving ${restored.length - currentUnmatched.length} missing-create blocker(s) after recovery sync; ` +
2508
- `chain snapshot did not account for the affected slot(s).`, 'warn');
2509
- }
804
+ return cowRuntime.preserveMissingCreateBlockersAfterRecovery(this, blockers, recoveryResult);
2510
805
  }
2511
806
  /**
2512
807
  * Merge missing CREATE result contexts into manager._lastUnmatchedChainOrders.
@@ -2520,42 +815,7 @@ class DEXBot {
2520
815
  * @returns {void}
2521
816
  */
2522
817
  _markMissingCreateResultsAsStructuralBlocker(missingCreateResults) {
2523
- const blockers = Array.isArray(missingCreateResults)
2524
- ? missingCreateResults.map(item => {
2525
- const order = item.ctx?.order || {};
2526
- const fingerprint = [
2527
- `type=${order.type || 'unknown'}`,
2528
- `price=${Format.formatPrice6(order.price)}`,
2529
- `size=${Format.formatAmount(order.size)}`
2530
- ].join(',');
2531
- return {
2532
- chainOrderId: 'unknown',
2533
- type: order.type || null,
2534
- price: order.price,
2535
- size: order.size,
2536
- slotId: order.id || item.ctx?.id || null,
2537
- reason: 'missing-create-result',
2538
- operationIndex: item.index,
2539
- fingerprint,
2540
- };
2541
- })
2542
- : [];
2543
- if (this.manager && blockers.length > 0) {
2544
- const existing = Array.isArray(this.manager._lastUnmatchedChainOrders)
2545
- ? this.manager._lastUnmatchedChainOrders
2546
- : [];
2547
- const keys = new Set(existing.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
2548
- const merged = [...existing];
2549
- for (const blocker of blockers) {
2550
- const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
2551
- if (!keys.has(key)) {
2552
- merged.push(blocker);
2553
- keys.add(key);
2554
- }
2555
- }
2556
- this.manager._lastUnmatchedChainOrders = merged;
2557
- this.manager._lastUnmatchedChainOrdersAt = Date.now();
2558
- }
818
+ return cowRuntime.markMissingCreateResultsAsStructuralBlocker(this, missingCreateResults);
2559
819
  }
2560
820
  /**
2561
821
  * Format an unmatched chain order/blocker for COW logs.
@@ -2564,7 +824,7 @@ class DEXBot {
2564
824
  * @returns {string} Compact human-readable diagnostic.
2565
825
  */
2566
826
  _formatUnmatchedChainOrderForLog(order) {
2567
- return formatUnmatchedChainOrder(order);
827
+ return cowRuntime.formatUnmatchedChainOrderForLog(order);
2568
828
  }
2569
829
  /**
2570
830
  * Record a pending CREATE broadcast on the manager.
@@ -2587,35 +847,7 @@ class DEXBot {
2587
847
  * @returns {void}
2588
848
  */
2589
849
  _recordPendingBroadcast(entry) {
2590
- if (!this.manager || !entry || !entry.order)
2591
- return;
2592
- if (!this.manager._pendingBroadcasts || !(this.manager._pendingBroadcasts instanceof Map)) {
2593
- this.manager._pendingBroadcasts = new Map();
2594
- }
2595
- const fingerprint = buildCreateOpFingerprint({
2596
- side: entry.order.type,
2597
- assetA: this.manager?.assets?.assetA?.id,
2598
- assetB: this.manager?.assets?.assetB?.id,
2599
- sellInt: entry.finalInts?.sell,
2600
- receiveInt: entry.finalInts?.receive,
2601
- slotId: entry.order.id
2602
- });
2603
- if (!fingerprint) {
2604
- this.manager.logger.log?.(`[COW] Skipped pending-broadcast record: could not build fingerprint for ${entry.order?.id || 'unknown'}`, 'warn');
2605
- return;
2606
- }
2607
- this.manager._pendingBroadcasts.set(fingerprint, {
2608
- fingerprint,
2609
- opIndex: entry.opIndex,
2610
- ctxIndex: entry.ctxIndex,
2611
- slotId: entry.order.id,
2612
- orderId: entry.order.id,
2613
- orderType: entry.order.type,
2614
- order: entry.order,
2615
- finalInts: entry.finalInts,
2616
- batchId: this._currentBatchId || null,
2617
- recordedAt: Date.now()
2618
- });
850
+ return cowRuntime.recordPendingBroadcast(this, entry);
2619
851
  }
2620
852
  /**
2621
853
  * Clear the pending-broadcast cache.
@@ -2626,9 +858,7 @@ class DEXBot {
2626
858
  * _reconcileAfterUncertainBroadcast before calling this).
2627
859
  */
2628
860
  _clearPendingBroadcasts() {
2629
- if (this.manager && this.manager._pendingBroadcasts instanceof Map) {
2630
- this.manager._pendingBroadcasts.clear();
2631
- }
861
+ return cowRuntime.clearPendingBroadcasts(this.manager?._pendingBroadcasts);
2632
862
  }
2633
863
  /**
2634
864
  * Build a fingerprint for an on-chain order so it can be matched against
@@ -2639,19 +869,7 @@ class DEXBot {
2639
869
  * @returns {string|null} Fingerprint or null on bad input
2640
870
  */
2641
871
  _buildChainOrderFingerprint(chainOrder, slotId) {
2642
- if (!chainOrder || !slotId)
2643
- return null;
2644
- const normalized = this._normalizeChainOrderForPendingMatch(chainOrder);
2645
- if (!normalized)
2646
- return null;
2647
- return buildCreateOpFingerprint({
2648
- side: normalized.side,
2649
- assetA: normalized.assetA,
2650
- assetB: normalized.assetB,
2651
- sellInt: normalized.sellInt,
2652
- receiveInt: normalized.receiveInt,
2653
- slotId
2654
- });
872
+ return cowRuntime.buildChainOrderFingerprint(this, chainOrder, slotId);
2655
873
  }
2656
874
  /**
2657
875
  * Normalize raw BitShares limit_order_object data into the integer tuple
@@ -2665,41 +883,7 @@ class DEXBot {
2665
883
  * @returns {{side: string, assetA: string, assetB: string, sellInt: number, receiveInt: number}|null}
2666
884
  */
2667
885
  _normalizeChainOrderForPendingMatch(chainOrder) {
2668
- if (!chainOrder)
2669
- return null;
2670
- const assetA = this.manager?.assets?.assetA?.id;
2671
- const assetB = this.manager?.assets?.assetB?.id;
2672
- if (!assetA || !assetB)
2673
- return null;
2674
- const explicitSide = (chainOrder.type === 'buy' || chainOrder.type === 'sell')
2675
- ? chainOrder.type
2676
- : null;
2677
- const explicitSell = chainOrder.sellInt ?? chainOrder.sell;
2678
- const explicitReceive = chainOrder.receiveInt ?? chainOrder.receive;
2679
- if (explicitSide && Number.isFinite(Number(explicitSell)) && Number.isFinite(Number(explicitReceive))) {
2680
- return {
2681
- side: explicitSide,
2682
- assetA,
2683
- assetB,
2684
- sellInt: Number(explicitSell),
2685
- receiveInt: Number(explicitReceive)
2686
- };
2687
- }
2688
- const base = chainOrder.sell_price?.base;
2689
- const quote = chainOrder.sell_price?.quote;
2690
- if (!base || !quote || !base.asset_id || !quote.asset_id)
2691
- return null;
2692
- const baseAmount = Number(base.amount);
2693
- const quoteAmount = Number(quote.amount);
2694
- if (!Number.isFinite(baseAmount) || !Number.isFinite(quoteAmount))
2695
- return null;
2696
- if (base.asset_id === assetA && quote.asset_id === assetB) {
2697
- return { side: 'sell', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
2698
- }
2699
- if (base.asset_id === assetB && quote.asset_id === assetA) {
2700
- return { side: 'buy', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
2701
- }
2702
- return null;
886
+ return cowRuntime.normalizeChainOrderForPendingMatch(this, chainOrder);
2703
887
  }
2704
888
  /**
2705
889
  * Find a chain order that matches a planned slot using price+size proximity.
@@ -2717,56 +901,7 @@ class DEXBot {
2717
901
  * @returns {Object|null} Matching chain order, or null
2718
902
  */
2719
903
  _findChainOrderForSlot(chainOrders, slotId, planned) {
2720
- if (!Array.isArray(chainOrders) || !slotId)
2721
- return null;
2722
- const assetA = this.manager?.assets?.assetA?.id;
2723
- const assetB = this.manager?.assets?.assetB?.id;
2724
- if (!assetA || !assetB)
2725
- return null;
2726
- // 1. Exact fingerprint match.
2727
- for (const o of chainOrders) {
2728
- const fp = this._buildChainOrderFingerprint(o, slotId);
2729
- if (fp && this.manager._pendingBroadcasts?.has(fp)) {
2730
- return o;
2731
- }
2732
- }
2733
- if (!planned || !Number.isFinite(Number(planned.sell)) || !Number.isFinite(Number(planned.receive))) {
2734
- return null;
2735
- }
2736
- // 2. Near match: same side, sell int within 1, receive int within 1% or 2 units.
2737
- const targetSell = Number(planned.sell);
2738
- const targetReceive = Number(planned.receive);
2739
- const plannedSide = planned.orderType ||
2740
- planned.side ||
2741
- this.manager._pendingBroadcasts?.get?.(planned.fingerprint)?.orderType ||
2742
- this.manager.orders.get(slotId)?.type;
2743
- if (plannedSide !== 'buy' && plannedSide !== 'sell') {
2744
- return null;
2745
- }
2746
- let best = null;
2747
- let bestDistance = Infinity;
2748
- for (const o of chainOrders) {
2749
- const normalized = this._normalizeChainOrderForPendingMatch(o);
2750
- if (!normalized)
2751
- continue;
2752
- if (normalized.side !== plannedSide)
2753
- continue;
2754
- const sell = Number(normalized.sellInt);
2755
- const receive = Number(normalized.receiveInt);
2756
- if (!Number.isFinite(sell) || !Number.isFinite(receive))
2757
- continue;
2758
- const sellDelta = Math.abs(sell - targetSell);
2759
- const receiveDelta = Math.abs(receive - targetReceive);
2760
- const receiveTol = Math.max(2, Math.floor(targetReceive * 0.01));
2761
- if (sellDelta > 1 || receiveDelta > receiveTol)
2762
- continue;
2763
- const distance = sellDelta * 1000 + receiveDelta;
2764
- if (distance < bestDistance) {
2765
- best = o;
2766
- bestDistance = distance;
2767
- }
2768
- }
2769
- return best;
904
+ return cowRuntime.findChainOrderForSlot(this, chainOrders, slotId, planned);
2770
905
  }
2771
906
  /**
2772
907
  * Reconcile a broadcast whose chain state is unknown.
@@ -2778,290 +913,27 @@ class DEXBot {
2778
913
  *
2779
914
  * Algorithm:
2780
915
  * 1. Read the account's current open orders from the chain.
2781
- * 2. For each pending-broadcast entry (fingerprinted CREATE op), look
2782
- * for a matching chain order. If found: adopt it (set the opContext's
2783
- * chainOrderId and continue with the existing planned slot).
2784
- * 3. For pending entries with no chain match: virtualize (mark the
2785
- * opContext as discarded; the planned slot stays empty until the
2786
- * next planning cycle).
2787
- * 4. Persist + log a structured [COW][UNCERTAIN] summary.
2788
- *
2789
- * After this method returns, the bot has either adopted the on-chain
2790
- * result (good case — chain accepted but we didn't see the reply) or
2791
- * accepted the discard (chain rejected or never received the op).
2792
- *
2793
- * @param {BroadcastUncertainError} err - The thrown error
2794
- * @param {Array<Object>} opContexts - Original opContexts from the failed batch
2795
- * @returns {Promise<Object>} Result object compatible with batch return shape
2796
- */
2797
- async _reconcileAfterUncertainBroadcast(err, opContexts, options = {}) {
2798
- if (this.manager?._fillProcessingLock &&
2799
- typeof this.manager._fillProcessingLock.acquire === 'function' &&
2800
- !this.manager._fillProcessingLock.isReentrant()) {
2801
- return this.manager._fillProcessingLock.acquire(() => this._reconcileAfterUncertainBroadcast(err, opContexts, options));
2802
- }
2803
- return this._reconcileAfterUncertainBroadcastImpl(err, opContexts, options);
2804
- }
2805
- async _reconcileAfterUncertainBroadcastImpl(err, opContexts, options) {
2806
- const startedAt = Date.now();
2807
- const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
2808
- ? Array.from(this.manager._pendingBroadcasts.values())
2809
- : [];
2810
- const createContextCount = opContexts.filter(c => c && c.kind === 'create').length;
2811
- const nonCreateContextCount = opContexts.length - createContextCount;
2812
- this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
2813
- `creates=${createContextCount} nonCreates=${nonCreateContextCount} ` +
2814
- `staleSinceMs=${err?.timeoutMs || 'n/a'}. Entering reconcile-then-decide.`, 'warn');
2815
- if (!chainOrders?.readOpenOrders) {
2816
- this.manager.logger.log('[COW][UNCERTAIN] readOpenOrders unavailable; falling back to structural resync only.', 'error');
2817
- if (typeof this.manager.requestStructuralGridResync === 'function') {
2818
- await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders unavailable', { batchId: err?.batchId || null });
2819
- }
2820
- this._clearPendingBroadcasts();
2821
- return { executed: false, hadRotation: false, uncertain: true };
2822
- }
2823
- // 1. Read the chain
2824
- const accountRef = this.accountId || this.account?.id || this.account;
2825
- let chainSnapshot = [];
2826
- try {
2827
- chainSnapshot = await chainOrders.readOpenOrders(accountRef);
2828
- }
2829
- catch (readErr) {
2830
- this.manager.logger.log(`[COW][UNCERTAIN] readOpenOrders failed: ${readErr?.message || readErr}. ` +
2831
- `Falling back to structural resync.`, 'error');
2832
- if (typeof this.manager.requestStructuralGridResync === 'function') {
2833
- await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders failed', { batchId: err?.batchId || null, error: readErr?.message || String(readErr) });
2834
- }
2835
- this._clearPendingBroadcasts();
2836
- return { executed: false, hadRotation: false, uncertain: true };
2837
- }
2838
- const adopted = [];
2839
- let discarded = [];
2840
- // 2. For each pending broadcast, look for a chain match.
2841
- for (const entry of pending) {
2842
- const match = this._findChainOrderForSlot(chainSnapshot, entry.slotId, {
2843
- sell: entry.finalInts?.sell,
2844
- receive: entry.finalInts?.receive,
2845
- orderType: entry.orderType || entry.order?.type,
2846
- fingerprint: entry.fingerprint
2847
- });
2848
- if (match) {
2849
- adopted.push({
2850
- slotId: entry.slotId,
2851
- chainOrderId: match.id,
2852
- orderType: entry.orderType || entry.order?.type,
2853
- });
2854
- this.manager._pendingBroadcasts.delete(entry.fingerprint);
2855
- }
2856
- else {
2857
- // Preserve full entry context so the recheck below can use
2858
- // both the fingerprint match path (against _pendingBroadcasts)
2859
- // and the near-match path (via finalInts/orderType), not just
2860
- // slotId alone.
2861
- discarded.push(entry);
2862
- }
2863
- }
2864
- // ---- RACE MITIGATION: block-delay recheck before discarding creates ----
2865
- // BROADCAST_DEADLINE fires while the transaction may still be valid
2866
- // and land in the next block. Wait for up to 3 blocks (~1 BitShares
2867
- // block interval each) and re-check before permanently discarding.
2868
- const UNCERTAIN_RECHECK_MAX_ATTEMPTS = this.config.pipelineTiming?.RETRY_MAX_ATTEMPTS ?? PIPELINE_TIMING.RETRY_MAX_ATTEMPTS; // 3
2869
- const UNCERTAIN_RECHECK_INTERVAL_MS = TIMING.MILLISECONDS_PER_SECOND * 3; // 3s
2870
- let recheckRound = 0;
2871
- while (discarded.length > 0 && recheckRound < UNCERTAIN_RECHECK_MAX_ATTEMPTS) {
2872
- recheckRound++;
2873
- await new Promise(r => setTimeout(r, UNCERTAIN_RECHECK_INTERVAL_MS));
2874
- let freshSnapshot;
2875
- try {
2876
- freshSnapshot = await chainOrders.readOpenOrders(accountRef);
2877
- }
2878
- catch {
2879
- break;
2880
- }
2881
- const stillDiscarded = [];
2882
- for (const entry of discarded) {
2883
- const match = this._findChainOrderForSlot(freshSnapshot, entry.slotId, {
2884
- sell: entry.finalInts?.sell,
2885
- receive: entry.finalInts?.receive,
2886
- orderType: entry.orderType || entry.order?.type,
2887
- fingerprint: entry.fingerprint
2888
- });
2889
- if (match) {
2890
- adopted.push({
2891
- slotId: entry.slotId,
2892
- chainOrderId: match.id,
2893
- orderType: entry.orderType || entry.order?.type,
2894
- });
2895
- this.manager.logger.log(`[COW][UNCERTAIN] CREATE re-adopted after ${recheckRound} block(s): ` +
2896
- `${entry.slotId}->${match.id}`, 'info');
2897
- }
2898
- else {
2899
- stillDiscarded.push(entry);
2900
- }
2901
- }
2902
- discarded = stillDiscarded;
2903
- }
2904
- // 3. Re-read chain with latest state. The initial snapshot may be
2905
- // stale for non-CREATE ops (updates/cancels) not tracked in
2906
- // _pendingBroadcasts — the seed cause of the slot-115 cascade.
2907
- let latestSnapshot;
2908
- try {
2909
- latestSnapshot = await chainOrders.readOpenOrders(accountRef);
2910
- }
2911
- catch {
2912
- latestSnapshot = chainSnapshot;
2913
- }
2914
- // Always sync to link adopted chain orders into grid slots.
2915
- // Otherwise an adopted slot stays VIRTUAL and the next COW cycle
2916
- // places a duplicate order at the same price.
2917
- let hadRotation = false;
2918
- if (latestSnapshot && latestSnapshot.length > 0 && this.manager?.syncFromOpenOrders) {
2919
- try {
2920
- await this.manager.syncFromOpenOrders(latestSnapshot, {
2921
- skipAccounting: true,
2922
- });
2923
- hadRotation = true;
2924
- }
2925
- catch (syncErr) {
2926
- this.manager.logger.log(`[COW][UNCERTAIN] syncFromOpenOrders failed during recovery: ${syncErr?.message || syncErr}`, 'error');
2927
- // Fallback: reload from persisted snapshot + re-sync.
2928
- this.manager.logger.log('[COW][UNCERTAIN] syncFromOpenOrders failed; falling back to full grid reload from persisted snapshot', 'warn');
2929
- const fallbackResult = await this._recoverFromPersistedGrid();
2930
- if (fallbackResult.success) {
2931
- hadRotation = true;
2932
- }
2933
- else {
2934
- this.manager.logger.log(`[COW][UNCERTAIN] Persisted grid reload failed: ${fallbackResult.reason}. ` +
2935
- `Escalating to structural resync.`, 'warn');
2936
- if (typeof this.manager.requestStructuralGridResync === 'function') {
2937
- this.manager.requestStructuralGridResync('cow-uncertain-recovery-failed', { reason: fallbackResult.reason }).catch((escalateErr) => {
2938
- this.manager.logger.log(`[COW][UNCERTAIN] Escalation to structural resync failed: ${escalateErr.message}`, 'error');
2939
- });
2940
- }
2941
- }
2942
- }
2943
- }
2944
- // 4. Log structured summary.
2945
- const elapsedMs = Date.now() - startedAt;
2946
- const heartbeatAgeMs = this._lastBroadcastHeartbeatAt
2947
- ? Date.now() - this._lastBroadcastHeartbeatAt
2948
- : null;
2949
- this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
2950
- `staleSinceMs=${err?.timeoutMs || 'n/a'} heartbeatAgeMs=${heartbeatAgeMs ?? 'n/a'} ` +
2951
- `adopted=${adopted.length} discarded=${discarded.length} recheckRounds=${recheckRound} elapsedMs=${elapsedMs}`, adopted.length > 0 ? 'info' : 'warn');
2952
- if (adopted.length > 0) {
2953
- this.manager.logger.log(`[COW][UNCERTAIN] Adopted chain orders: ${adopted
2954
- .map(a => `${a.slotId}->${a.chainOrderId}`)
2955
- .join(', ')}`, 'info');
2956
- }
2957
- if (discarded.length > 0) {
2958
- this.manager.logger.log(`[COW][UNCERTAIN] Discarded planned CREATEs (no chain match after ${recheckRound} recheck(s)): ${discarded
2959
- .map(d => d.slotId)
2960
- .join(', ')}`, 'warn');
2961
- // Restore target grid sizes for discarded CREATEs so the slots are
2962
- // not left as virtual/0 after recovery. Without this, the slot stays
2963
- // at size 0 (set by the fill handler) and is never reactivated until
2964
- // a fresh fill cycle triggers another rebalance that happens to succeed.
2965
- // entry.order is the pending-broadcast target order (captured at broadcast
2966
- // time); its .id matches entry.slotId — the lookup below keys on slotId.
2967
- for (const entry of discarded) {
2968
- if (entry.order && entry.slotId) {
2969
- const current = this.manager.orders.get(entry.slotId);
2970
- if (current && current.state === ORDER_STATES.VIRTUAL && !current.orderId) {
2971
- try {
2972
- await this.manager._applyOrderUpdate({ ...entry.order, state: ORDER_STATES.VIRTUAL, orderId: null }, 'uncertain-recovery-restore-size', { skipAccounting: true, fee: 0 });
2973
- this.manager.logger.log(`[COW][UNCERTAIN] Restored target size for discarded CREATE slot ${entry.slotId} (size: ${entry.order.size})`, 'info');
2974
- }
2975
- catch (restoreErr) {
2976
- this.manager.logger.log(`[COW][UNCERTAIN] Failed to restore size for slot ${entry.slotId}: ${restoreErr?.message || restoreErr}`, 'warn');
2977
- }
2978
- }
2979
- }
2980
- }
2981
- }
2982
- // ---- Structural resync safeguard after skipAccounting restore ----
2983
- // The discarded CREATE restore above used skipAccounting: true to avoid
2984
- // double-counting when the structural resync recalculates accounting
2985
- // from scratch. Ensure one is scheduled — if already in-flight (timer
2986
- // or running), the existing resync will handle the accounting fix.
2987
- if (discarded.length > 0 && typeof this.manager?.requestStructuralGridResync === 'function') {
2988
- const alreadyScheduled = this._structuralGridResyncRunning || this._structuralGridResyncTimer;
2989
- if (!alreadyScheduled) {
2990
- this.manager.logger.log(`[COW][UNCERTAIN] Scheduling structural resync after discarded CREATE restore ` +
2991
- `(skipAccounting used — resync needed for fund recalculation)`, 'info');
2992
- this.manager.requestStructuralGridResync('cow-uncertain-accounting-repair', { discardedCount: discarded.length }).catch((err) => {
2993
- this.manager.logger.log(`[COW][UNCERTAIN] Failed to schedule accounting repair resync: ${err.message}`, 'error');
2994
- });
2995
- }
2996
- else {
2997
- this.manager.logger.log(`[COW][UNCERTAIN] Structural resync already ${this._structuralGridResyncRunning ? 'running' : 'scheduled'}; ` +
2998
- `it will repair accounting after discarded CREATE restore`, 'debug');
2999
- }
3000
- }
3001
- this._clearPendingBroadcasts();
3002
- // Post-recovery safety net: if there are still unmatched chain
3003
- // orders after reconcile-then-decide, cancel ONE per cycle. The cap
3004
- // is enforced inside the helper via _autoCancelOrphanCycleMarker.
3005
- try {
3006
- const autoCancelResult = await this._autoCancelOneUnmatchedOrphan();
3007
- if (autoCancelResult.cancelled) {
3008
- this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancelled orphan ${autoCancelResult.orderId} ` +
3009
- `after recovery (cap=1/cycle).`, 'info');
3010
- }
3011
- }
3012
- catch (orphanErr) {
3013
- this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancel pass failed: ${orphanErr?.message || orphanErr}`, 'warn');
3014
- }
3015
- // Boundary shift recovery: the COW batch that should have committed the
3016
- // boundary shift from this fill cycle failed before commit. Only COUNT
3017
- // ADOPTED CREATEs — orders that actually landed on-chain. Discarded
3018
- // CREATEs represent orders that never existed, so the grid must NOT
3019
- // shift the boundary for them. The next fill cycle's
3020
- // calculateTargetGrid → deriveTargetBoundary will recompute the correct
3021
- // boundary from scratch for any discarded slots.
3022
- //
3023
- // NOTE: Direct mutation of manager.boundaryIdx outside a COW commit
3024
- // violates the invariant stated in grid.ts:1217-1220 (boundary must only
3025
- // be updated atomically inside _commitWorkingGrid). This is intentional
3026
- // here because the COW commit already failed — the invariant was already
3027
- // broken by the broadcast uncertainty, and the adjustment only restores
3028
- // the state to what the successful commit would have produced for
3029
- // orders that are confirmed on-chain.
3030
- if (hadRotation && this.manager && adopted.length > 0) {
3031
- let boundaryShift = 0;
3032
- for (const entry of adopted) {
3033
- if (entry.orderType === ORDER_TYPES.SELL) {
3034
- boundaryShift--; // SELL CREATE → fill was BUY → boundary LEFT
3035
- }
3036
- else if (entry.orderType === ORDER_TYPES.BUY) {
3037
- boundaryShift++; // BUY CREATE → fill was SELL → boundary RIGHT
3038
- }
3039
- }
3040
- if (boundaryShift !== 0) {
3041
- const oldIdx = this.manager.boundaryIdx;
3042
- if (typeof oldIdx === 'number') {
3043
- const maxIdx = Math.max(0, (this.manager.orders?.size || 1) - 1);
3044
- const newIdx = Math.max(0, Math.min(maxIdx, oldIdx + boundaryShift));
3045
- if (newIdx !== oldIdx) {
3046
- this.manager.boundaryIdx = newIdx;
3047
- this.manager.logger.log(`[COW][UNCERTAIN] Boundary adjusted by ${boundaryShift} ` +
3048
- `(${adopted.length} adopted CREATE(s), ${discarded.length} discarded): ` +
3049
- `${oldIdx} → ${newIdx}. Next COW cycle will generate replacement opposite-side orders.`, 'warn');
3050
- if (typeof this.manager._markGridDirty === 'function') {
3051
- this.manager._markGridDirty();
3052
- }
3053
- }
3054
- }
3055
- }
3056
- }
3057
- // Persist master grid mutations from the reconciliation sync and any
3058
- // orphan auto-cancellation that ran above. These apply outside the COW
3059
- // broadcast path and would otherwise be in-memory only. The boundary
3060
- // adjustment above is also persisted here.
3061
- if (typeof this.manager?.persistGrid === 'function') {
3062
- await this.manager.persistGrid();
3063
- }
3064
- return { executed: false, hadRotation, uncertain: true, adopted, discarded };
916
+ * 2. For each pending-broadcast entry (fingerprinted CREATE op), look
917
+ * for a matching chain order. If found: adopt it (set the opContext's
918
+ * chainOrderId and continue with the existing planned slot).
919
+ * 3. For pending entries with no chain match: virtualize (mark the
920
+ * opContext as discarded; the planned slot stays empty until the
921
+ * next planning cycle).
922
+ * 4. Persist + log a structured [COW][UNCERTAIN] summary.
923
+ *
924
+ * After this method returns, the bot has either adopted the on-chain
925
+ * result (good case — chain accepted but we didn't see the reply) or
926
+ * accepted the discard (chain rejected or never received the op).
927
+ *
928
+ * @param {BroadcastUncertainError} err - The thrown error
929
+ * @param {Array<Object>} opContexts - Original opContexts from the failed batch
930
+ * @returns {Promise<Object>} Result object compatible with batch return shape
931
+ */
932
+ async _reconcileAfterUncertainBroadcast(err, opContexts, options = {}) {
933
+ return cowRuntime.reconcileAfterUncertainBroadcast(this, err, opContexts, options);
934
+ }
935
+ async _reconcileAfterUncertainBroadcastImpl(err, opContexts, options) {
936
+ return cowRuntime.reconcileAfterUncertainBroadcastImpl(this, err, opContexts, options);
3065
937
  }
3066
938
  /**
3067
939
  * Auto-cancel a price-drift orphan from the unmatched-order snapshot.
@@ -3089,72 +961,7 @@ class DEXBot {
3089
961
  * @returns {Promise<{cancelled: boolean, orderId?: string, reason?: string}>}
3090
962
  */
3091
963
  async _autoCancelOneUnmatchedOrphan() {
3092
- const cycleId = this._currentCycleId || 0;
3093
- // FIX 4: Higher cap during recovery mode so orphan backlog clears faster.
3094
- const recoveryActive = this.manager?._recoveryState?.structuralResyncRequested === true;
3095
- const cycleCap = recoveryActive ? 5 : 1;
3096
- if (this._autoCancelOrphanCycleMarker === cycleId) {
3097
- if (this._autoCancelOrphanSubCount >= cycleCap) {
3098
- return { cancelled: false, reason: 'cap-reached-this-cycle', subCount: this._autoCancelOrphanSubCount };
3099
- }
3100
- }
3101
- else {
3102
- // Reset sub-count on first call this cycle
3103
- this._autoCancelOrphanCycleMarker = cycleId;
3104
- this._autoCancelOrphanSubCount = 0;
3105
- }
3106
- const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
3107
- ? this.manager._pendingBroadcasts.size
3108
- : 0;
3109
- if (pending > 0) {
3110
- return { cancelled: false, reason: 'pending-broadcasts-active' };
3111
- }
3112
- const unmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
3113
- ? this.manager._lastUnmatchedChainOrders
3114
- : [];
3115
- if (unmatched.length === 0) {
3116
- return { cancelled: false, reason: 'no-unmatched' };
3117
- }
3118
- // Check for fingerprinted entries first — these came from a pending
3119
- // broadcast (missing-create-result) and must be handled by the recovery
3120
- // path, not by auto-cancel. The first fingerprinted entry is checked
3121
- // regardless of its reason field.
3122
- const fingerprinted = unmatched.find(u => u && u.fingerprint);
3123
- if (fingerprinted) {
3124
- return { cancelled: false, reason: 'fingerprinted-handle-via-recovery' };
3125
- }
3126
- // Only cancel price-drift orphans — these are surplus orders that
3127
- // drifted away from their assigned slot price and have no adoptable
3128
- // grid slot. All other unmatched orders (duplicate-price-level,
3129
- // already-matched-slot, etc.) are adoptable positions that the structural
3130
- // resync will integrate into the grid — cancelling them destroys capital.
3131
- const target = unmatched.find(u => u && u.reason === 'price-drift-orphan');
3132
- if (!target) {
3133
- return { cancelled: false, reason: 'no-price-drift-orphan', message: 'no price-drift orphan to cancel; other unmatched orders are adoptable' };
3134
- }
3135
- const orderId = target.id || target.orderId || target.chainOrderId;
3136
- if (!orderId) {
3137
- return { cancelled: false, reason: 'no-orderId' };
3138
- }
3139
- if (!chainOrders?.cancelOrder) {
3140
- return { cancelled: false, reason: 'cancelOrder-unavailable' };
3141
- }
3142
- try {
3143
- await chainOrders.cancelOrder(this.account, this.privateKey, orderId);
3144
- if (typeof chainOrders.recordOwnCancel === 'function') {
3145
- chainOrders.recordOwnCancel(orderId);
3146
- }
3147
- // Increment sub-count only after a confirmed successful cancel.
3148
- // Failed cancels do not consume the per-cycle cap budget.
3149
- this._autoCancelOrphanSubCount++;
3150
- this.manager.logger.log(`[COW] Auto-cancelled ${this._autoCancelOrphanSubCount}/${unmatched.length} unmatched chain order ` +
3151
- `(${this._formatUnmatchedChainOrderForLog(target)}) — per-cycle cap=${cycleCap}.`, 'warn');
3152
- return { cancelled: true, orderId };
3153
- }
3154
- catch (err) {
3155
- this.manager.logger.log(`[COW] Auto-cancel of unmatched chain order ${orderId} failed: ${err?.message || err}`, 'error');
3156
- return { cancelled: false, reason: 'cancel-failed', error: err?.message || String(err) };
3157
- }
964
+ return cowRuntime.autoCancelOneUnmatchedOrphan(this);
3158
965
  }
3159
966
  // Pair mode applies only when create contexts include both BUY and SELL.
3160
967
  // Single-side create batches intentionally remain a single executeBatch.
@@ -3164,21 +971,7 @@ class DEXBot {
3164
971
  * @returns {boolean} True if pair mode should be used
3165
972
  */
3166
973
  _shouldExecuteCreatePairMode(opContexts) {
3167
- if (!Array.isArray(opContexts) || opContexts.length < 2)
3168
- return false;
3169
- if (!opContexts.every(ctx => ctx?.kind === 'create' && ctx?.order))
3170
- return false;
3171
- let hasBuy = false;
3172
- let hasSell = false;
3173
- for (const ctx of opContexts) {
3174
- if (ctx.order.type === ORDER_TYPES.BUY)
3175
- hasBuy = true;
3176
- if (ctx.order.type === ORDER_TYPES.SELL)
3177
- hasSell = true;
3178
- if (hasBuy && hasSell)
3179
- return true;
3180
- }
3181
- return false;
974
+ return cowRuntime.shouldExecuteCreatePairMode(this, opContexts);
3182
975
  }
3183
976
  /**
3184
977
  * Execute operations with retry on BroadcastUncertainError.
@@ -3190,36 +983,7 @@ class DEXBot {
3190
983
  * the full operations array would duplicate those creates on chain.
3191
984
  */
3192
985
  async _executeWithRetryOnUncertain(operations, opContexts) {
3193
- const MAX_RETRIES = 1;
3194
- for (let attempt = 1;; attempt++) {
3195
- try {
3196
- return await this._executeOperationsWithStrategy(operations, opContexts);
3197
- }
3198
- catch (err) {
3199
- const isRetriable = err instanceof BroadcastUncertainError
3200
- && !err.partialOnChainState
3201
- && attempt <= MAX_RETRIES;
3202
- if (isRetriable) {
3203
- this.manager.logger.log(`[COW] Broadcast uncertain (attempt ${attempt}/${MAX_RETRIES + 1}), retrying...`, 'warn');
3204
- await this._ensureCredentialDaemonWritable('COW batch retry');
3205
- // Reconcile before retry — stale ops would fail "not found".
3206
- try {
3207
- const accountRef = this.accountId || this.account?.id || this.account;
3208
- const freshChain = await chainOrders.readOpenOrders(accountRef);
3209
- if (freshChain.length > 0 && this.manager?.syncFromOpenOrders) {
3210
- await this.manager.syncFromOpenOrders(freshChain, {
3211
- skipAccounting: true,
3212
- });
3213
- }
3214
- }
3215
- catch (syncErr) {
3216
- this.manager.logger.log(`[COW] Pre-retry sync failed (non-fatal): ${syncErr?.message || syncErr}`, 'warn');
3217
- }
3218
- continue;
3219
- }
3220
- throw err;
3221
- }
3222
- }
986
+ return cowRuntime.executeWithRetryOnUncertain(this, operations, opContexts);
3223
987
  }
3224
988
  /**
3225
989
  * Execute blockchain operations with appropriate strategy (single batch or pair mode).
@@ -3228,63 +992,7 @@ class DEXBot {
3228
992
  * @returns {Promise<{result: Object, opContexts: Array}>} Execution result with contexts
3229
993
  */
3230
994
  async _executeOperationsWithStrategy(operations, opContexts) {
3231
- if (!this._shouldExecuteCreatePairMode(opContexts)) {
3232
- const result = await chainOrders.executeBatch(this.account, this.privateKey, operations);
3233
- return { result, opContexts };
3234
- }
3235
- const createEntries = [];
3236
- for (let i = 0; i < operations.length; i++) {
3237
- createEntries.push({
3238
- operation: operations[i],
3239
- context: opContexts[i],
3240
- });
3241
- }
3242
- const groups = this._buildOutsideInPairGroupsForCreateEntries(createEntries);
3243
- const mergedOperationResults = [];
3244
- const mergedRawResults = [];
3245
- const mergedContexts = [];
3246
- // Grouped execution is fail-fast, but NOT atomic across groups.
3247
- // Each group is a separate on-chain transaction; if a later group fails,
3248
- // earlier groups may already be confirmed on-chain.
3249
- for (let idx = 0; idx < groups.length; idx++) {
3250
- const group = groups[idx];
3251
- const groupOps = group.map(e => e.operation);
3252
- const groupContexts = group.map(e => e.context);
3253
- this.manager.logger.log(`[COW] Broadcasting create pair group ${idx + 1}/${groups.length} (${groupOps.length} op${groupOps.length > 1 ? 's' : ''}, outside->center)`, 'info');
3254
- let groupResult;
3255
- try {
3256
- groupResult = await chainOrders.executeBatch(this.account, this.privateKey, groupOps);
3257
- }
3258
- catch (err) {
3259
- const groupsBroadcast = idx;
3260
- const groupsTotal = groups.length;
3261
- const broadcastedOperationCount = mergedContexts.length;
3262
- this.manager.logger.log(`[COW] Grouped create execution failed at group ${idx + 1}/${groupsTotal}; ${groupsBroadcast} group(s) already broadcast (${broadcastedOperationCount} op context(s)). Partial on-chain state is possible.`, 'error');
3263
- err.partialOnChainState = groupsBroadcast > 0;
3264
- err.groupsBroadcast = groupsBroadcast;
3265
- err.groupsTotal = groupsTotal;
3266
- err.broadcastedOperationCount = broadcastedOperationCount;
3267
- throw err;
3268
- }
3269
- const groupOpResults = this._extractOperationResults(groupResult);
3270
- mergedOperationResults.push(...groupOpResults);
3271
- mergedRawResults.push(groupResult?.raw || null);
3272
- mergedContexts.push(...groupContexts);
3273
- }
3274
- return {
3275
- result: {
3276
- success: true,
3277
- raw: {
3278
- grouped: true,
3279
- groupsExecuted: groups.length,
3280
- groupResults: mergedRawResults,
3281
- },
3282
- operation_results: mergedOperationResults,
3283
- grouped: true,
3284
- groupsExecuted: groups.length
3285
- },
3286
- opContexts: mergedContexts
3287
- };
995
+ return cowRuntime.executeOperationsWithStrategy(this, operations, opContexts);
3288
996
  }
3289
997
  /**
3290
998
  * Validate that operations can be executed with available funds before broadcasting.
@@ -3296,94 +1004,7 @@ class DEXBot {
3296
1004
  * @private
3297
1005
  */
3298
1006
  _validateOperationFunds(operations, assetA, assetB) {
3299
- if (!operations || operations.length === 0) {
3300
- return { isValid: true, summary: 'No operations to validate' };
3301
- }
3302
- const { blockchainToFloat, floatToBlockchainInt, quantizeFloat } = require('./order/utils/math');
3303
- const snap = this.manager.getChainFundsSnapshot();
3304
- const netRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
3305
- const runningRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
3306
- const peakRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
3307
- // Sum amounts and check individual order sizes
3308
- for (const op of operations) {
3309
- if (!op?.op_data)
3310
- continue;
3311
- let sellAssetId = null;
3312
- let sellAmountInt = 0;
3313
- if (op.op_name === 'limit_order_create') {
3314
- sellAssetId = op.op_data.amount_to_sell?.asset_id;
3315
- sellAmountInt = op.op_data.amount_to_sell?.amount;
3316
- }
3317
- else if (op.op_name === 'limit_order_update') {
3318
- // In limit_order_update, new_price.base is the amount to sell
3319
- sellAssetId = op.op_data.new_price?.base?.asset_id;
3320
- sellAmountInt = op.op_data.new_price?.base?.amount;
3321
- }
3322
- if (sellAssetId && (sellAmountInt !== undefined && sellAmountInt !== null)) {
3323
- const precision = (sellAssetId === assetA.id) ? assetA.precision : assetB.precision;
3324
- const assetSymbol = (sellAssetId === assetA.id) ? assetA.symbol : assetB.symbol;
3325
- // CRITICAL SAFETY CHECK: Ensure amount is greater than zero
3326
- if (Number(sellAmountInt) <= 0) {
3327
- return {
3328
- isValid: false,
3329
- summary: `[VALIDATION] CRITICAL: Zero amount order detected for ${assetSymbol} (assetId=${sellAssetId})`,
3330
- violations: [{ asset: assetSymbol, sizeInt: sellAmountInt, reason: 'Zero amount' }]
3331
- };
3332
- }
3333
- // Track signed per-op deltas in operation order.
3334
- // CREATE consumes full amount; UPDATE consumes/releases delta.
3335
- let signedDelta = 0;
3336
- if (op.op_name === 'limit_order_update') {
3337
- const deltaAssetId = op.op_data.delta_amount_to_sell?.asset_id;
3338
- const deltaSellInt = op.op_data.delta_amount_to_sell?.amount;
3339
- if (deltaAssetId === sellAssetId && Number.isFinite(Number(deltaSellInt))) {
3340
- signedDelta = blockchainToFloat(deltaSellInt, precision);
3341
- }
3342
- }
3343
- else {
3344
- signedDelta = blockchainToFloat(sellAmountInt, precision);
3345
- }
3346
- netRequiredFunds[sellAssetId] = quantizeFloat((netRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
3347
- runningRequiredFunds[sellAssetId] = quantizeFloat((runningRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
3348
- const nextPeak = Math.max(Number(peakRequiredFunds[sellAssetId] || 0), Number(runningRequiredFunds[sellAssetId] || 0));
3349
- peakRequiredFunds[sellAssetId] = quantizeFloat(nextPeak, precision);
3350
- }
3351
- }
3352
- // Calculate available funds - CRITICAL FIX: Check against FREE balance, not free+required
3353
- // Bug: Previous logic added requiredFunds to available, making validation meaningless
3354
- // Correct logic: available = chainFree (current free balance)
3355
- // If required > available, batch will fail on execution
3356
- const availableFunds = {
3357
- [assetA.id]: quantizeFloat(snap.chainFreeSell || 0, assetA.precision),
3358
- [assetB.id]: quantizeFloat(snap.chainFreeBuy || 0, assetB.precision)
3359
- };
3360
- // Check for fund violations using quantized comparison
3361
- const fundViolations = [];
3362
- for (const assetId in peakRequiredFunds) {
3363
- const required = peakRequiredFunds[assetId];
3364
- const netRequired = netRequiredFunds[assetId] || 0;
3365
- const available = availableFunds[assetId] || 0;
3366
- // Use precision-aware comparison
3367
- const prec = (assetId === assetA.id) ? assetA.precision : assetB.precision;
3368
- if (floatToBlockchainInt(required, prec) > floatToBlockchainInt(available, prec)) {
3369
- fundViolations.push({
3370
- asset: assetId === assetA.id ? assetA.symbol : assetB.symbol,
3371
- required,
3372
- netRequired,
3373
- available,
3374
- deficit: quantizeFloat(required - available, prec)
3375
- });
3376
- }
3377
- }
3378
- if (fundViolations.length > 0) {
3379
- let summary = `[VALIDATION] Fund validation FAILED:\n`;
3380
- for (const v of fundViolations) {
3381
- summary += ` ${v.asset}: peakRequired=${Format.formatAmount8(v.required)}, netRequired=${Format.formatAmount8(v.netRequired)}, available=${Format.formatAmount8(v.available)}, deficit=${Format.formatAmount8(v.deficit)}\n`;
3382
- }
3383
- return { isValid: false, summary: summary.trim(), violations: fundViolations };
3384
- }
3385
- const summary = `[VALIDATION] PASSED: ${operations.length} operations`;
3386
- return { isValid: true, summary };
1007
+ return cowRuntime.validateOperationFunds(this, operations, assetA, assetB);
3387
1008
  }
3388
1009
  /**
3389
1010
  * Resolve the ideal size from an order-like object with fallback.
@@ -3392,20 +1013,7 @@ class DEXBot {
3392
1013
  * @returns {number|null} Resolved size or null
3393
1014
  */
3394
1015
  _resolveIdealSizeForValidation(orderLike, fallbackSize = null) {
3395
- const candidates = [
3396
- orderLike?.idealSize,
3397
- orderLike?.order?.idealSize,
3398
- orderLike?.size,
3399
- orderLike?.order?.size,
3400
- fallbackSize
3401
- ];
3402
- for (const candidate of candidates) {
3403
- const numeric = Number(candidate);
3404
- if (Number.isFinite(numeric) && numeric > 0) {
3405
- return numeric;
3406
- }
3407
- }
3408
- return null;
1016
+ return cowRuntime.resolveIdealSizeForValidation(this, orderLike, fallbackSize);
3409
1017
  }
3410
1018
  /**
3411
1019
  * Validate that an order size is safe to execute (above minimum dust thresholds).
@@ -3416,7 +1024,7 @@ class DEXBot {
3416
1024
  * @returns {import('./types').OrderValidationResult}
3417
1025
  */
3418
1026
  _validateOrderSizeForExecution(size, type, orderLike = null, fallbackSize = null) {
3419
- return validateOrderSize(size, type, this.manager.assets, this.config.gridLimits?.MIN_ORDER_SIZE_FACTOR, this._resolveIdealSizeForValidation(orderLike, fallbackSize), this.config.gridLimits?.PARTIAL_DUST_THRESHOLD_PERCENTAGE);
1027
+ return cowRuntime.validateOrderSizeForExecution(this, size, type, orderLike, fallbackSize);
3420
1028
  }
3421
1029
  /**
3422
1030
  * Execute a batch of order operations if the rebalance result has executable actions.
@@ -3756,77 +1364,7 @@ class DEXBot {
3756
1364
  * @returns {Array<{type: string, id: string, order?: Object, orderId?: string, newSize?: number, newPrice?: number, newGridId?: string}>}
3757
1365
  */
3758
1366
  _buildActionsFromPlan(plan) {
3759
- const normalizedPlan = Array.isArray(plan)
3760
- ? { ordersToPlace: plan }
3761
- : (plan || {});
3762
- const { ordersToPlace = [], ordersToRotate = [], ordersToUpdate = [], ordersToCancel = [] } = normalizedPlan;
3763
- const actions = [];
3764
- for (const o of ordersToCancel) {
3765
- if (o?.orderId) {
3766
- actions.push({ type: COW_ACTIONS.CANCEL, id: o.id, orderId: o.orderId });
3767
- }
3768
- }
3769
- for (const r of ordersToRotate) {
3770
- const oldOrder = r?.oldOrder || r;
3771
- const id = oldOrder?.id || r?.id;
3772
- const orderId = oldOrder?.orderId || r?.orderId;
3773
- const newGridId = r?.newGridId || id;
3774
- const newSize = Number.isFinite(Number(r?.newSize))
3775
- ? Number(r.newSize)
3776
- : Number(r?.size || oldOrder?.size || 0);
3777
- const newPrice = Number.isFinite(Number(r?.newPrice))
3778
- ? Number(r.newPrice)
3779
- : Number(r?.price || oldOrder?.price);
3780
- const orderType = r?.type || oldOrder?.type;
3781
- if (!id || !orderId || !newGridId || !orderType || !Number.isFinite(newPrice) || !(newSize > 0))
3782
- continue;
3783
- actions.push({
3784
- type: COW_ACTIONS.UPDATE,
3785
- id,
3786
- orderId,
3787
- newGridId,
3788
- newSize,
3789
- newPrice,
3790
- order: {
3791
- id: newGridId,
3792
- type: orderType,
3793
- price: newPrice,
3794
- size: newSize
3795
- }
3796
- });
3797
- }
3798
- for (const o of ordersToUpdate) {
3799
- const partialOrder = o?.partialOrder || o;
3800
- const id = o?.id || partialOrder?.id;
3801
- const orderId = o?.orderId || partialOrder?.orderId;
3802
- const orderType = o?.type || partialOrder?.type;
3803
- const newSize = Number.isFinite(Number(o?.newSize))
3804
- ? Number(o.newSize)
3805
- : Number(partialOrder?.size || 0);
3806
- if (!id || !orderId)
3807
- continue;
3808
- actions.push({
3809
- type: COW_ACTIONS.UPDATE,
3810
- id,
3811
- orderId,
3812
- newSize,
3813
- order: {
3814
- ...(partialOrder || {}),
3815
- id,
3816
- orderId,
3817
- type: orderType,
3818
- size: newSize
3819
- }
3820
- });
3821
- }
3822
- // Run CREATE actions after UPDATE actions so same-batch downsizes can
3823
- // release balance before placements consume it.
3824
- for (const o of ordersToPlace) {
3825
- if (!o?.id)
3826
- continue;
3827
- actions.push({ type: COW_ACTIONS.CREATE, id: o.id, order: o });
3828
- }
3829
- return actions;
1367
+ return cowRuntime.buildActionsFromPlan(this, plan);
3830
1368
  }
3831
1369
  /**
3832
1370
  * Build a COW result object (workingGrid + actions) from a simple plan.
@@ -3834,78 +1372,7 @@ class DEXBot {
3834
1372
  * @returns {{workingGrid: import('./types').WorkingGrid, workingIndexes: Object, workingBoundary: number, actions: Array}}
3835
1373
  */
3836
1374
  _buildCowResultFromPlan(plan) {
3837
- const { WorkingGrid } = require('./order/working_grid');
3838
- const workingGrid = new WorkingGrid(this.manager.orders, {
3839
- baseVersion: Number.isFinite(Number(this.manager._gridVersion)) ? this.manager._gridVersion : 0
3840
- });
3841
- const workingBoundary = this.manager.boundaryIdx;
3842
- const actions = this._buildActionsFromPlan(plan);
3843
- // Project planned actions into working grid so COW commit carries intended transitions.
3844
- for (const action of actions) {
3845
- if (action.type === COW_ACTIONS.CANCEL) {
3846
- const current = workingGrid.get(action.id);
3847
- if (!current)
3848
- continue;
3849
- workingGrid.set(action.id, convertToSpreadPlaceholder(current));
3850
- }
3851
- else if (action.type === COW_ACTIONS.CREATE) {
3852
- if (!action.id || !action.order)
3853
- continue;
3854
- const current = workingGrid.get(action.id) || { id: action.id };
3855
- workingGrid.set(action.id, {
3856
- ...current,
3857
- ...action.order,
3858
- id: action.id,
3859
- state: ORDER_STATES.VIRTUAL,
3860
- orderId: null
3861
- });
3862
- }
3863
- else if (action.type === COW_ACTIONS.UPDATE) {
3864
- if (action.newGridId && action.newGridId !== action.id) {
3865
- const current = workingGrid.get(action.id);
3866
- if (current) {
3867
- workingGrid.set(action.id, convertToSpreadPlaceholder(current));
3868
- }
3869
- const targetId = action.newGridId;
3870
- const targetCurrent = workingGrid.get(targetId) || { id: targetId };
3871
- const rotatedSize = Number.isFinite(Number(action.newSize))
3872
- ? Number(action.newSize)
3873
- : Number(targetCurrent.size || 0);
3874
- const rotatedPrice = Number.isFinite(Number(action.newPrice))
3875
- ? Number(action.newPrice)
3876
- : Number(action.order?.price ?? targetCurrent.price);
3877
- workingGrid.set(targetId, {
3878
- ...targetCurrent,
3879
- ...(action.order || {}),
3880
- id: targetId,
3881
- size: rotatedSize,
3882
- price: rotatedPrice,
3883
- state: ORDER_STATES.VIRTUAL,
3884
- orderId: null
3885
- });
3886
- continue;
3887
- }
3888
- const current = workingGrid.get(action.id);
3889
- if (!current)
3890
- continue;
3891
- const newSize = Number.isFinite(Number(action.newSize))
3892
- ? Number(action.newSize)
3893
- : Number(current.size || 0);
3894
- workingGrid.set(action.id, {
3895
- ...current,
3896
- ...(action.order || {}),
3897
- id: action.id,
3898
- orderId: action.orderId || current.orderId,
3899
- size: newSize
3900
- });
3901
- }
3902
- }
3903
- return {
3904
- workingGrid,
3905
- workingIndexes: workingGrid.getIndexes(),
3906
- workingBoundary,
3907
- actions
3908
- };
1375
+ return cowRuntime.buildCowResultFromPlan(this, plan);
3909
1376
  }
3910
1377
  /**
3911
1378
  * Restore skipped update slots in the working grid to master state.
@@ -3915,16 +1382,7 @@ class DEXBot {
3915
1382
  * @returns {void}
3916
1383
  */
3917
1384
  _restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedSlotIds, skippedCount = 0) {
3918
- if (!workingGrid || !skippedSlotIds || skippedSlotIds.size === 0) {
3919
- return;
3920
- }
3921
- const masterVersion = Number.isFinite(Number(this.manager?._gridVersion))
3922
- ? Number(this.manager._gridVersion)
3923
- : undefined;
3924
- for (const slotId of skippedSlotIds) {
3925
- workingGrid.syncFromMaster(this.manager.orders, slotId, masterVersion);
3926
- }
3927
- this.manager.logger.log(`[COW] Restored ${skippedSlotIds.size} slot(s) after ${skippedCount} skipped update action(s).`, 'debug');
1385
+ return cowRuntime.restoreSkippedUpdateSlotsInWorkingGrid(this, workingGrid, skippedSlotIds, skippedCount);
3928
1386
  }
3929
1387
  /**
3930
1388
  * COW broadcast: Execute blockchain operations and commit working grid on success.
@@ -3934,751 +1392,10 @@ class DEXBot {
3934
1392
  * @private
3935
1393
  */
3936
1394
  async _updateOrdersOnChainBatchCOW(cowResult) {
3937
- this._currentCycleId = (Number.isFinite(Number(this._currentCycleId)) ? Number(this._currentCycleId) : 0) + 1;
3938
- const { workingGrid, workingIndexes, workingBoundary, actions } = cowResult;
3939
- if (this.config.dryRun) {
3940
- const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
3941
- const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
3942
- const updateCount = actions.filter(a => a.type === COW_ACTIONS.UPDATE).length;
3943
- if (cancelCount > 0)
3944
- this.manager.logger.log(`Dry run: would cancel ${cancelCount} orders`, 'info');
3945
- if (createCount > 0)
3946
- this.manager.logger.log(`Dry run: would place ${createCount} new orders`, 'info');
3947
- if (updateCount > 0)
3948
- this.manager.logger.log(`Dry run: would update ${updateCount} orders`, 'info');
3949
- return { executed: true, hadRotation: false };
3950
- }
3951
- const createSlotValidation = validateCreateTargetSlots(actions, this.manager?.orders);
3952
- if (!createSlotValidation.isValid) {
3953
- for (const violation of createSlotValidation.violations) {
3954
- this.manager.logger.log(`[COW] Rejecting CREATE for occupied slot ${violation.targetId}: ` +
3955
- `existing orderId=${violation.currentOrderId}, type=${violation.currentType}, state=${violation.currentState}`, 'error');
3956
- }
3957
- return {
3958
- executed: false,
3959
- aborted: true,
3960
- reason: 'CREATE_SLOT_OCCUPIED',
3961
- violations: createSlotValidation.violations,
3962
- hadRotation: false
3963
- };
3964
- }
3965
- const hasCreateActions = actions.some(action => action.type === COW_ACTIONS.CREATE);
3966
- // Gap 5: Recovery exhausted — block all CREATES. The bot has exhausted
3967
- // its recovery attempt budget and needs a fill or sync cycle to reset.
3968
- // Existing orders remain active and are monitored, but no new orders
3969
- // are placed until recovery resets.
3970
- if (hasCreateActions && this.manager?._recoveryExhaustedAt) {
3971
- const exhaustedAge = Date.now() - this.manager._recoveryExhaustedAt;
3972
- this.manager.logger.log?.(`[RECOVERY-EXHAUSTED] Blocking ${actions.filter(a => a.type === COW_ACTIONS.CREATE).length} CREATE(s) ` +
3973
- `(exhausted ${(exhaustedAge / 1000).toFixed(0)}s ago). ` +
3974
- `Waiting for next fill or sync cycle to reset recovery state.`, 'warn');
3975
- return {
3976
- executed: false,
3977
- aborted: true,
3978
- reason: 'RECOVERY_EXHAUSTED',
3979
- hadRotation: false
3980
- };
3981
- }
3982
- const unmatchedChainOrders = Array.isArray(this.manager?._lastUnmatchedChainOrders)
3983
- ? this.manager._lastUnmatchedChainOrders
3984
- : [];
3985
- // PENDING_BROADCASTS: an earlier batch timed out at the credential
3986
- // daemon and the chain status of its CREATEs is still unknown. The
3987
- // recovery path (_reconcileAfterUncertainBroadcast) is responsible
3988
- // for resolving it. We refuse to publish a fresh CREATE batch until
3989
- // recovery runs, otherwise we risk double-publishing or stacking
3990
- // orphan orders on top of potentially-orphaned ones.
3991
- const pendingBroadcasts = (this.manager && this.manager._pendingBroadcasts instanceof Map)
3992
- ? Array.from(this.manager._pendingBroadcasts.values())
3993
- : [];
3994
- if (hasCreateActions && (unmatchedChainOrders.length > 0 || pendingBroadcasts.length > 0)) {
3995
- // ---- Handle pending broadcasts first ----
3996
- if (pendingBroadcasts.length > 0) {
3997
- this.manager.logger.log(`[COW] Rejecting CREATE batch: ${pendingBroadcasts.length} pending broadcast(s) from a prior uncertain ` +
3998
- `broadcast. Running recovery before placing replacement orders.`, 'error');
3999
- if (typeof this.manager.requestStructuralGridResync === 'function') {
4000
- if (this.manager._recoveryState)
4001
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
4002
- await this.manager.requestStructuralGridResync('pending broadcasts before COW create', { pendingBroadcasts: pendingBroadcasts.map(p => p.slotId) });
4003
- }
4004
- try {
4005
- await this._reconcileAfterUncertainBroadcast(new BroadcastUncertainError('rejected CREATE batch had pending broadcasts', {
4006
- operations: pendingBroadcasts.map(p => p.order),
4007
- accountName: this.account,
4008
- batchId: this._currentBatchId || null,
4009
- payload: null,
4010
- timeoutMs: null
4011
- }), []);
4012
- }
4013
- catch (recoverErr) {
4014
- this.manager.logger.log(`[COW] Recovery from pending broadcasts failed: ${recoverErr?.message || recoverErr}`, 'error');
4015
- }
4016
- return {
4017
- executed: false,
4018
- aborted: true,
4019
- reason: 'PENDING_BROADCASTS',
4020
- hadRotation: false
4021
- };
4022
- }
4023
- // ---- Unmatched orders: adopt instead of cancel ----
4024
- // Unmatched orders are legitimate on-chain positions the grid
4025
- // hasn't adopted yet. Cancelling them destroys capital and
4026
- // creates gaps in the order book. Instead, re-sync to adopt
4027
- // them into the grid, then reject (the sync invalidated the
4028
- // working grid, so the existing COW plan is stale).
4029
- // NOTE: syncFromOpenOrders may be a partial no-op under certain
4030
- // lock conditions (e.g. _fillProcessingLock + !isReentrant).
4031
- // Structural resync scheduled below handles adoption regardless.
4032
- const unmatchedSample = unmatchedChainOrders
4033
- .slice(0, 3)
4034
- .map(o => this._formatUnmatchedChainOrderForLog(o))
4035
- .join(' | ');
4036
- this.manager.logger.log(`[COW] ${unmatchedChainOrders.length} unmatched chain order(s) blocking CREATES ` +
4037
- (unmatchedSample ? `(${unmatchedSample})` : '') +
4038
- ` — adopting via sync instead of cancelling`, 'info');
4039
- try {
4040
- const accountRef = this.account;
4041
- const freshSnapshot = await chainOrders.readOpenOrders(accountRef);
4042
- if (freshSnapshot && freshSnapshot.length > 0) {
4043
- const syncResult = await this.manager.syncFromOpenOrders(freshSnapshot, {
4044
- skipAccounting: true,
4045
- });
4046
- // Only overwrite _lastUnmatchedChainOrders when sync actually
4047
- // processed orders (filled + updated + corrected > 0).
4048
- // Force-release and lock-contention early-exit paths return
4049
- // empty arrays without touching _lastUnmatchedChainOrders —
4050
- // overwriting with [] would drop stale unmatched entries.
4051
- if (syncResult && Array.isArray(syncResult.unmatchedChainOrders)) {
4052
- const processed = (syncResult.filledOrders?.length || 0) +
4053
- (syncResult.updatedOrders?.length || 0) +
4054
- (syncResult.ordersNeedingCorrection?.length || 0);
4055
- if (processed > 0) {
4056
- this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders;
4057
- this.manager.logger.log(`[COW] Adopted chain order(s) via sync: ${processed} processed, ` +
4058
- `${syncResult.unmatchedChainOrders.length} still unmatched`, 'info');
4059
- }
4060
- else {
4061
- // processed === 0 with unmatched still present means the
4062
- // sync could not adopt the unmatched chain orders. This is
4063
- // normal when called re-entrantly (inside _fillProcessingLock):
4064
- // the sync engine runs inline and either timed out (unlikely
4065
- // for a re-entrant call) or all chain orders were already
4066
- // matched — leaving only stale unmatched entries.
4067
- const syncUnmatchedCount = syncResult.unmatchedChainOrders.length;
4068
- this.manager.logger.log(`[COW] Sync returned without processing (processed=0, ` +
4069
- `unmatched=${syncUnmatchedCount} in result, ` +
4070
- `_lastUnmatchedChainOrders=${unmatchedChainOrders.length}). ` +
4071
- `Structural resync will handle adoption.`, syncUnmatchedCount > 0 ? 'warn' : 'debug');
4072
- // If the sync result itself has unmatched entries that
4073
- // differ from _lastUnmatchedChainOrders, adopt them now
4074
- // so the stale tracker is more accurate for the resync.
4075
- if (syncUnmatchedCount > 0 && syncUnmatchedCount !== unmatchedChainOrders.length) {
4076
- this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders.map((o) => ({ ...o }));
4077
- this.manager.logger.log(`[COW] Updated _lastUnmatchedChainOrders from sync result: ` +
4078
- `${unmatchedChainOrders.length} → ${syncUnmatchedCount}`, 'debug');
4079
- }
4080
- }
4081
- }
4082
- }
4083
- }
4084
- catch (syncErr) {
4085
- this.manager.logger.log(`[COW] Failed to sync/unmatched orders: ${syncErr?.message || syncErr}`, 'warn');
4086
- }
4087
- // Working grid is stale after master grid mutation from sync.
4088
- // Request structural resync to rebuild the grid on the next cycle.
4089
- if (typeof this.manager.requestStructuralGridResync === 'function') {
4090
- if (this.manager._recoveryState)
4091
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
4092
- await this.manager.requestStructuralGridResync('unmatched chain orders before COW create', { unmatchedChainOrders: unmatchedChainOrders });
4093
- }
4094
- this.manager.logger.log(`[COW] Rejecting CREATE batch after sync: working grid invalidated by master mutation`, 'info');
4095
- return {
4096
- executed: false,
4097
- aborted: true,
4098
- reason: 'UNMATCHED_CHAIN_ORDERS',
4099
- hadRotation: false
4100
- };
4101
- }
4102
- const { assetA, assetB } = this.manager.assets;
4103
- const operations = [];
4104
- const opContexts = [];
4105
- const skippedUpdateSlotIds = new Set();
4106
- let skippedUpdateCount = 0;
4107
- // Collect IDs to lock from actions
4108
- const idsToLock = new Set();
4109
- for (const action of actions) {
4110
- if (action.type === COW_ACTIONS.CANCEL && action.orderId) {
4111
- idsToLock.add(action.orderId);
4112
- if (action.id)
4113
- idsToLock.add(action.id);
4114
- }
4115
- else if (action.type === COW_ACTIONS.CREATE && action.id) {
4116
- idsToLock.add(action.id);
4117
- }
4118
- else if (action.type === COW_ACTIONS.UPDATE && action.orderId) {
4119
- idsToLock.add(action.orderId);
4120
- if (action.id)
4121
- idsToLock.add(action.id);
4122
- }
4123
- }
4124
- // Apply shadow locks
4125
- this.manager.lockOrders(idsToLock);
4126
- try {
4127
- this._batchInFlight = true;
4128
- this._markGridActivity('batch start');
4129
- this.manager._setRebalanceState(REBALANCE_STATES.BROADCASTING);
4130
- this.manager.startBroadcasting();
4131
- // Build operations from actions
4132
- for (const action of actions) {
4133
- if (action.type === COW_ACTIONS.CANCEL) {
4134
- try {
4135
- const op = await chainOrders.buildCancelOrderOp(this.account, action.orderId);
4136
- operations.push(op);
4137
- const order = this.manager.orders.get(action.id) || { id: action.id, orderId: action.orderId };
4138
- opContexts.push({ kind: 'cancel', order });
4139
- }
4140
- catch (err) {
4141
- const orderNotFound = /\bnot found\b/i.test(err.message) || /\bdoes not exist\b/i.test(err.message);
4142
- if (orderNotFound) {
4143
- this.manager.logger.log(`[COW] Cancel skipped for ${action.id} (${action.orderId}): order already removed from chain`, 'debug');
4144
- }
4145
- else {
4146
- this.manager.logger.log(`Failed to prepare cancel op for ${action.id}: ${err.message}`, 'error');
4147
- }
4148
- }
4149
- }
4150
- else if (action.type === COW_ACTIONS.CREATE) {
4151
- try {
4152
- const order = action.order;
4153
- const sizeValidation = this._validateOrderSizeForExecution(order.size, order.type, order, order.size);
4154
- if (!sizeValidation.isValid) {
4155
- this.manager.logger.log(`Skipping create op for ${action.id}: ${sizeValidation.reason}`, 'warn');
4156
- continue;
4157
- }
4158
- const liveSlot = this.manager.orders.get(order.id);
4159
- const plannedPrice = Number(order.price);
4160
- const livePrice = liveSlot ? Number(liveSlot.price) : NaN;
4161
- const priceDrift = Number.isFinite(plannedPrice) && Number.isFinite(livePrice)
4162
- ? Math.abs(livePrice - plannedPrice)
4163
- : 0;
4164
- const effectiveOrder = (priceDrift > 0)
4165
- ? { ...order, price: livePrice, size: order.size, type: order.type }
4166
- : order;
4167
- if (priceDrift > 0) {
4168
- this.manager.logger.log(`[COW] Pre-broadcast price freshness: slot ${order.id} ` +
4169
- `drifted from planned=${plannedPrice} to live=${livePrice} ` +
4170
- `(diff=${priceDrift}); rebuilding CREATE op with live price.`, 'debug');
4171
- }
4172
- const args = buildCreateOrderArgs(effectiveOrder, assetA, assetB);
4173
- const buildResult = await chainOrders.buildCreateOrderOp(this.account, args.amountToSell, args.sellAssetId, args.minToReceive, args.receiveAssetId, null);
4174
- if (!buildResult) {
4175
- this.manager.logger.log(`Skipping create op for ${action.id}: amounts would round to 0 on blockchain`, 'warn');
4176
- continue;
4177
- }
4178
- operations.push(buildResult.op);
4179
- opContexts.push({ kind: 'create', id: order.id, order: effectiveOrder, args, finalInts: buildResult.finalInts });
4180
- this._recordPendingBroadcast({
4181
- opIndex: operations.length - 1,
4182
- ctxIndex: opContexts.length - 1,
4183
- order: effectiveOrder,
4184
- finalInts: buildResult.finalInts
4185
- });
4186
- }
4187
- catch (err) {
4188
- this.manager.logger.log(`Failed to prepare create op for ${action.id}: ${err.message}`, 'error');
4189
- }
4190
- }
4191
- else if (action.type === COW_ACTIONS.UPDATE) {
4192
- try {
4193
- // Rotation update: move existing on-chain order to a new slot/price.
4194
- if (action.newGridId && action.newGridId !== action.id) {
4195
- const masterOrder = this.manager.orders.get(action.id);
4196
- const orderType = action.order?.type || masterOrder?.type;
4197
- const newPrice = Number.isFinite(Number(action.newPrice))
4198
- ? Number(action.newPrice)
4199
- : Number(action.order?.price);
4200
- const newSize = Number.isFinite(Number(action.newSize))
4201
- ? Number(action.newSize)
4202
- : Number(action.order?.size || 0);
4203
- if (!masterOrder || !action.orderId || !orderType || !Number.isFinite(newPrice) || newSize <= 0) {
4204
- continue;
4205
- }
4206
- const rotationSizeValidation = this._validateOrderSizeForExecution(newSize, orderType, action.order, newSize);
4207
- if (!rotationSizeValidation.isValid) {
4208
- this.manager.logger.log(`Skipping rotation update ${action.id} -> ${action.newGridId}: ${rotationSizeValidation.reason}`, 'warn');
4209
- continue;
4210
- }
4211
- const { amountToSell, minToReceive } = buildCreateOrderArgs({ type: orderType, size: newSize, price: newPrice }, assetA, assetB);
4212
- const buildResult = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell, minToReceive, newPrice, orderType }, masterOrder.rawOnChain || null);
4213
- if (!buildResult) {
4214
- skippedUpdateCount++;
4215
- if (action.id)
4216
- skippedUpdateSlotIds.add(action.id);
4217
- if (action.newGridId)
4218
- skippedUpdateSlotIds.add(action.newGridId);
4219
- this.manager.logger.log(`[COW] Skipping rotation update ${action.id} -> ${action.newGridId}: no blockchain delta`, 'debug');
4220
- continue;
4221
- }
4222
- operations.push(buildResult.op);
4223
- opContexts.push({
4224
- kind: 'rotation',
4225
- rotation: {
4226
- oldOrder: { ...masterOrder },
4227
- newGridId: action.newGridId,
4228
- newPrice,
4229
- newSize,
4230
- type: orderType
4231
- },
4232
- finalInts: buildResult.finalInts
4233
- });
4234
- continue;
4235
- }
4236
- const newSize = Number.isFinite(Number(action.newSize))
4237
- ? Number(action.newSize)
4238
- : Number(action.order?.size || 0);
4239
- const masterOrder = this.manager.orders.get(action.id);
4240
- const orderType = action.order?.type || masterOrder?.type;
4241
- const cachedRawOnChain = masterOrder?.rawOnChain || action.order?.rawOnChain || null;
4242
- const op = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell: newSize, orderType }, cachedRawOnChain);
4243
- if (!op) {
4244
- skippedUpdateCount++;
4245
- if (action.id)
4246
- skippedUpdateSlotIds.add(action.id);
4247
- if (action.newGridId)
4248
- skippedUpdateSlotIds.add(action.newGridId);
4249
- this.manager.logger.log(`[COW] Skipping size update ${action.id} (${action.orderId}): no blockchain delta`, 'debug');
4250
- continue;
4251
- }
4252
- operations.push(op.op);
4253
- const partialOrder = masterOrder || {
4254
- id: action.id,
4255
- orderId: action.orderId,
4256
- type: orderType
4257
- };
4258
- opContexts.push({ kind: 'size-update', updateInfo: { partialOrder, newSize }, finalInts: op.finalInts });
4259
- // Catch for both rotation-update and size-update branches.
4260
- }
4261
- catch (err) {
4262
- const orderNotFound = /\bnot found\b/i.test(err.message) || /\bdoes not exist\b/i.test(err.message);
4263
- if (orderNotFound) {
4264
- try {
4265
- const fbOrder = action.order || this.manager.orders.get(action.id);
4266
- const fbType = fbOrder?.type;
4267
- const fbSize = action.newSize || fbOrder?.size || 0;
4268
- // Live price from TARGET slot (same pattern as CREATE path).
4269
- const targetSlotId = action.newGridId || action.id;
4270
- const plannedPrice = action.newPrice || action.order?.price || 0;
4271
- const liveSlotForPrice = this.manager.orders.get(targetSlotId);
4272
- const livePrice = liveSlotForPrice ? Number(liveSlotForPrice.price) : NaN;
4273
- const priceDrift = Number.isFinite(plannedPrice) && Number.isFinite(livePrice)
4274
- ? Math.abs(livePrice - plannedPrice)
4275
- : 0;
4276
- const fbPrice = (priceDrift > 0) ? livePrice : plannedPrice;
4277
- if (priceDrift > 0) {
4278
- this.manager.logger.log(`[COW] CREATE fallback price drift for ${action.id} -> ${targetSlotId}: ` +
4279
- `planned=${plannedPrice} live=${livePrice} (diff=${priceDrift})`, 'debug');
4280
- }
4281
- // Same size gate as regular CREATE path.
4282
- const sizeCheck = this._validateOrderSizeForExecution(fbSize, fbType, fbOrder, fbSize);
4283
- if (!sizeCheck.isValid) {
4284
- this.manager.logger.log(`[COW] CREATE fallback for ${action.id} rejected by size validation: ${sizeCheck.reason}`, 'warn');
4285
- }
4286
- else if (fbType && fbSize > 0 && fbPrice > 0) {
4287
- const fbArgs = buildCreateOrderArgs({ type: fbType, size: fbSize, price: fbPrice }, assetA, assetB);
4288
- const fbResult = await chainOrders.buildCreateOrderOp(this.account, fbArgs.amountToSell, fbArgs.sellAssetId, fbArgs.minToReceive, fbArgs.receiveAssetId, null);
4289
- if (fbResult) {
4290
- operations.push(fbResult.op);
4291
- opContexts.push({
4292
- kind: 'create',
4293
- id: targetSlotId,
4294
- order: { id: targetSlotId, type: fbType, price: fbPrice, size: fbSize },
4295
- args: { amountToSell: fbArgs.amountToSell, minToReceive: fbArgs.minToReceive },
4296
- finalInts: fbResult.finalInts
4297
- });
4298
- this._recordPendingBroadcast({
4299
- opIndex: operations.length - 1,
4300
- ctxIndex: opContexts.length - 1,
4301
- order: { id: targetSlotId, type: fbType, price: fbPrice, size: fbSize },
4302
- finalInts: fbResult.finalInts
4303
- });
4304
- this.manager.logger.log(`[COW] Recovered "not found" for ${action.id}: converted UPDATE to CREATE for slot ${targetSlotId}`, 'warn');
4305
- continue;
4306
- }
4307
- }
4308
- }
4309
- catch (fbErr) {
4310
- this.manager.logger.log(`[COW] CREATE fallback also failed for ${action.id}: ${fbErr.message}`, 'warn');
4311
- }
4312
- }
4313
- this.manager.logger.log(`Failed to prepare update op for ${action.id}: ${err.message}`, 'error');
4314
- }
4315
- }
4316
- }
4317
- if (skippedUpdateCount > 0) {
4318
- this._restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedUpdateSlotIds, skippedUpdateCount);
4319
- }
4320
- if (operations.length === 0) {
4321
- this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
4322
- return { executed: false, hadRotation: false };
4323
- }
4324
- // Validate funds before broadcasting
4325
- const validation = this._validateOperationFunds(operations, assetA, assetB);
4326
- this.manager.logger.log(validation.summary, validation.isValid ? 'info' : 'warn');
4327
- if (!validation.isValid) {
4328
- this.manager.logger.log(`Skipping batch broadcast: ${validation.violations.length} fund violation(s) detected`, 'warn');
4329
- this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
4330
- return { executed: false, hadRotation: false };
4331
- }
4332
- await this._ensureCredentialDaemonWritable('COW batch broadcast');
4333
- // Execute batch
4334
- this.manager.logger.log(`[COW] Broadcasting batch with ${operations.length} operations...`, 'info');
4335
- this._lastBroadcastHeartbeatAt = Date.now();
4336
- const execution = await this._executeWithRetryOnUncertain(operations, opContexts);
4337
- const result = execution.result;
4338
- const executedContexts = execution.opContexts;
4339
- // Process results and commit on success
4340
- this.manager.pauseFundRecalc();
4341
- try {
4342
- this.manager._throwOnIllegalState = true;
4343
- if (result.success) {
4344
- // Pre-commit integrity: only CREATE ops require returned chainOrderIds.
4345
- // Cancel/update operation results may be empty depending on the broadcaster.
4346
- const preCommitResults = this._extractOperationResults(result, 'pre-commit-integrity');
4347
- const missingCreateResults = this._findMissingCreateResultContexts(preCommitResults, executedContexts);
4348
- if (missingCreateResults.length > 0) {
4349
- const missingSlots = missingCreateResults
4350
- .map(item => item.ctx?.order?.id || item.ctx?.id || `op-${item.index}`)
4351
- .join(', ');
4352
- this.manager.logger.log(`[COW] Refusing to commit working grid: ${missingCreateResults.length} CREATE op(s) ` +
4353
- `returned no chainOrderId (${missingSlots}). Discarding working grid and syncing from chain.`, 'error');
4354
- this.manager._clearWorkingGridRef();
4355
- this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
4356
- this._markMissingCreateResultsAsStructuralBlocker(missingCreateResults);
4357
- await this._recoverAfterMissingCreateResults('missing create operation results');
4358
- return {
4359
- executed: false,
4360
- hadRotation: false,
4361
- missingCreateResults: missingCreateResults.map(item => ({
4362
- index: item.index,
4363
- slotId: item.ctx?.order?.id || item.ctx?.id || null
4364
- }))
4365
- };
4366
- }
4367
- // SUCCESS: Commit working grid to master (atomic swap)
4368
- this.manager.logger.log('[COW] Blockchain success - committing working grid to master', 'info');
4369
- // RC-FIX: skipRecalc prevents invariant violation before optimistic accounting
4370
- await this.manager._commitWorkingGrid(workingGrid, workingIndexes, workingBoundary, { skipRecalc: true });
4371
- // Commitment accounting is handled in real-time by
4372
- // updateOptimisticFreeBalance when capital is committed to orders.
4373
- // The old post-batch deduction path was removed to avoid double-counting.
4374
- // Process batch results for logging/metrics
4375
- const batchResult = await this._processBatchResults(result, executedContexts);
4376
- // Persist to disk. CRITICAL: working grid was already committed
4377
- // to master above; if persistence is skipped or validation fails
4378
- // here, the on-disk snapshot will be older than the in-memory
4379
- // master. Retry once and surface the failure explicitly so the
4380
- // next cycle / shutdown can recover.
4381
- const persistResult = await this.manager.persistGrid();
4382
- if (persistResult && (persistResult.skipped || persistResult.isValid === false)) {
4383
- this.manager.logger.log(`[COW][PERSIST-GUARD] First persist attempt was ` +
4384
- `${persistResult.skipped ? 'skipped' : 'invalid'} ` +
4385
- `(${persistResult.reason || 'no reason'}); retrying once before ` +
4386
- `clearing working grid reference.`, 'warn');
4387
- this.manager._persistenceWarning = persistResult;
4388
- const retryResult = await this.manager.persistGrid();
4389
- if (retryResult && (retryResult.skipped || retryResult.isValid === false)) {
4390
- this.manager.logger.log(`[COW][PERSIST-GUARD] Retry also skipped/invalid ` +
4391
- `(${retryResult.reason || 'no reason'}). Master grid in memory ` +
4392
- `is ahead of disk snapshot; structural resync requested.`, 'error');
4393
- if (typeof this.manager.requestStructuralGridResync === 'function') {
4394
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
4395
- await this.manager.requestStructuralGridResync('persistence guard triggered after COW batch', { persistReason: retryResult.reason || 'unknown' });
4396
- }
4397
- }
4398
- else {
4399
- delete this.manager._persistenceWarning;
4400
- }
4401
- }
4402
- else if (this.manager._persistenceWarning) {
4403
- delete this.manager._persistenceWarning;
4404
- }
4405
- this._metrics.batchesExecuted++;
4406
- this.manager._clearWorkingGridRef();
4407
- this._clearPendingBroadcasts();
4408
- return { executed: true, hadRotation: true, ...batchResult };
4409
- }
4410
- else {
4411
- // FAILURE: Working grid discarded, master unchanged
4412
- this.manager.logger.log('[COW] Blockchain failed - working grid discarded, master unchanged', 'warn');
4413
- this.manager._clearWorkingGridRef();
4414
- this._clearPendingBroadcasts();
4415
- return { executed: false, hadRotation: false, ...result };
4416
- }
4417
- }
4418
- finally {
4419
- this.manager._throwOnIllegalState = false;
4420
- // Keep broadcasting true during resumeFundRecalc to skip invariant checks
4421
- // that would fail due to stale accountTotals (not yet refreshed from blockchain)
4422
- await this.manager.resumeFundRecalc();
4423
- this.manager.stopBroadcasting();
4424
- const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
4425
- const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
4426
- this.manager.logger.logFundsStatus(this.manager, `AFTER COW batch (created=${createCount}, cancelled=${cancelCount})`);
4427
- }
4428
- }
4429
- catch (err) {
4430
- this.manager.logger.log(`[COW] Batch transaction failed: ${err.message}`, 'error');
4431
- if (err?.partialOnChainState) {
4432
- this.manager.logger.log(`[COW] Non-atomic grouped execution detected (${err.groupsBroadcast}/${err.groupsTotal} groups broadcast). Local rollback cannot undo confirmed on-chain operations; next sync/reconcile will converge state.`, 'warn');
4433
- }
4434
- this.manager.stopBroadcasting();
4435
- this.manager._clearWorkingGridRef();
4436
- // BROADCAST_UNCERTAIN: the credential daemon timed out (or hit its
4437
- // inner deadline) and the chain status of the planned CREATEs is
4438
- // unknown. Run the reconcile-then-decide recovery path: read the
4439
- // chain, match each pending broadcast by fingerprint, adopt any
4440
- // chain-side matches, and discard the rest. We MUST NOT throw —
4441
- // throwing would re-enter the catch loop on the next attempt and
4442
- // potentially double-publish.
4443
- if (err instanceof BroadcastUncertainError) {
4444
- return await this._reconcileAfterUncertainBroadcast(err, opContexts);
4445
- }
4446
- // Handle hard abort
4447
- const hardAbortResult = await this._handleBatchHardAbort(err, 'COW batch processing', operations.length);
4448
- if (hardAbortResult)
4449
- return hardAbortResult;
4450
- // Check for stale orders — filled in the ~1.5s broadcast window after our plan was built.
4451
- const staleOrderIds = new Set();
4452
- const patterns = [
4453
- /Limit order (1\.7\.\d+) does not exist/g,
4454
- /Unable to find Object (1\.7\.\d+)/g,
4455
- /object (1\.7\.\d+) (?:does not exist|not found)/gi
4456
- ];
4457
- for (const pattern of patterns) {
4458
- let m;
4459
- while ((m = pattern.exec(err.message)) !== null) {
4460
- staleOrderIds.add(m[1]);
4461
- }
4462
- }
4463
- // "Cannot deduct all or more from order than order contains" means the order still
4464
- // exists, but its on-chain size shrank during the broadcast window. That is not an
4465
- // explicit stale/missing-order signal, so reconcile from chain instead of virtualizing.
4466
- if (/Cannot deduct all or more from order than order contains/.test(err.message)) {
4467
- return await this._recoverBatchSizeDrift(err, opContexts);
4468
- }
4469
- if (staleOrderIds.size > 0) {
4470
- // Recover explicit missing-order failures without aborting the entire fill cycle.
4471
- // Use the manager update path so indexes, working-grid sync, accounting, and
4472
- // persistence stay coherent.
4473
- return await this._recoverExplicitStaleOrders(staleOrderIds, 'cow-stale-order-cleanup');
4474
- }
4475
- throw err;
4476
- }
4477
- finally {
4478
- this._batchInFlight = false;
4479
- this._markGridActivity('batch end');
4480
- this.manager.unlockOrders(idsToLock);
4481
- if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
4482
- this._scheduleFillConsumerRestart(chainOrders);
4483
- }
4484
- }
1395
+ return cowRuntime.updateOrdersOnChainBatchCOW(this, cowResult);
4485
1396
  }
4486
- /**
4487
- * Process results from batch transaction execution.
4488
- * Updates order state, synchronizes with chain, and deducts BTS fees.
4489
- * @param {Object} result - Transaction result from executeBatch
4490
- * @param {Array} opContexts - Operation context array with operation metadata (must be 1:1 with result.operation_results)
4491
- * @returns {Object} Result with { executed: boolean, hadRotation: boolean }
4492
- * @private
4493
- */
4494
1397
  async _processBatchResults(result, opContexts) {
4495
- const results = this._extractOperationResults(result, '_processBatchResults');
4496
- const { getAssetFees } = require('./order/utils/math');
4497
- // IMPORTANT: Call without amount to get fee schedule fields
4498
- // ({ createFee, updateFee, ... }), not proceeds projection fields.
4499
- const btsFeeData = getAssetFees('BTS');
4500
- let hadRotation = false;
4501
- let updateOperationCount = 0;
4502
- const updatesToApply = [];
4503
- for (let i = 0; i < opContexts.length; i++) {
4504
- const ctx = opContexts[i];
4505
- const res = results[i];
4506
- if (ctx.kind === 'cancel') {
4507
- this.manager.logger.log(`Cancelled surplus order ${ctx.order.id} (${ctx.order.orderId})`, 'info');
4508
- const oldOrder = ctx.order;
4509
- const committedOrder = oldOrder?.id ? this.manager.orders.get(oldOrder.id) : null;
4510
- // The COW commit already updated manager.orders. Apply ONLY the optimistic
4511
- // accounting transition using pre-commit -> committed order states.
4512
- if (oldOrder && committedOrder && this.manager.accountant) {
4513
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, committedOrder, 'fill-cancel', btsFeeData?.cancelFee || 0, false);
4514
- }
4515
- }
4516
- else if (ctx.kind === 'size-update') {
4517
- const oldOrder = ctx.updateInfo.partialOrder;
4518
- const ord = this.manager.orders.get(oldOrder.id);
4519
- // Apply optimistic accounting from pre-commit -> committed state,
4520
- // including blockchain update fee deduction.
4521
- if (oldOrder && ord && this.manager.accountant) {
4522
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
4523
- }
4524
- if (ord) {
4525
- const updatedSlot = { ...ord, size: ctx.updateInfo.newSize };
4526
- // Update rawOnChain cache with new integers
4527
- if (ctx.finalInts) {
4528
- updatedSlot.rawOnChain = {
4529
- id: ord.orderId,
4530
- for_sale: String(ctx.finalInts.sell),
4531
- sell_price: {
4532
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4533
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4534
- }
4535
- };
4536
- }
4537
- updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
4538
- }
4539
- this.manager.logger.log(`Size update complete: ${ctx.updateInfo.partialOrder.orderId}`, 'info');
4540
- updateOperationCount++;
4541
- }
4542
- else if (ctx.kind === 'create') {
4543
- const chainOrderId = res && res[1];
4544
- if (chainOrderId) {
4545
- // synchronizeWithChain handles the full VIRTUAL -> ACTIVE transition
4546
- // including orderId assignment and fee deduction.
4547
- await this.manager.synchronizeWithChain({
4548
- gridOrderId: ctx.order.id, chainOrderId, expectedType: ctx.order.type, fee: btsFeeData.createFee
4549
- }, 'createOrder');
4550
- // After sync, apply rawOnChain metadata if available
4551
- if (ctx.finalInts) {
4552
- const syncedOrder = this.manager.orders.get(ctx.order.id);
4553
- if (syncedOrder) {
4554
- updatesToApply.push({
4555
- order: {
4556
- ...syncedOrder,
4557
- rawOnChain: {
4558
- id: chainOrderId,
4559
- for_sale: String(ctx.finalInts.sell),
4560
- sell_price: {
4561
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4562
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4563
- }
4564
- }
4565
- },
4566
- context: 'post-placement-metadata'
4567
- });
4568
- }
4569
- }
4570
- this.manager.logger.log(`Placed ${ctx.order.type} order ${ctx.order.id} -> ${chainOrderId}`, 'info');
4571
- }
4572
- else {
4573
- const fingerprint = [
4574
- `type=${ctx.order.type || 'unknown'}`,
4575
- `price=${Format.formatPrice6(ctx.order.price)}`,
4576
- `size=${Format.formatAmount(ctx.order.size)}`
4577
- ].join(',');
4578
- this.manager.logger.log(`[COW] CRITICAL: Create op for slot ${ctx.order.id} (type=${ctx.order.type}) ` +
4579
- `returned no chainOrderId. Identify any orphaned on-chain order by local fingerprint ` +
4580
- `${fingerprint} before cancelling.`, 'error');
4581
- }
4582
- }
4583
- else if (ctx.kind === 'rotation') {
4584
- hadRotation = true;
4585
- const { rotation } = ctx;
4586
- const { oldOrder, newPrice, newGridId, newSize, type } = rotation;
4587
- if (!newGridId) {
4588
- // Size correction only
4589
- const ord = this.manager.orders.get(oldOrder.id || rotation.id);
4590
- // Apply optimistic accounting from pre-commit -> committed state,
4591
- // including blockchain update fee deduction.
4592
- if (oldOrder && ord && this.manager.accountant) {
4593
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
4594
- }
4595
- if (ord) {
4596
- const updatedSlot = { ...ord, size: newSize };
4597
- // Update rawOnChain cache with new integers
4598
- if (ctx.finalInts) {
4599
- updatedSlot.rawOnChain = {
4600
- id: ord.orderId,
4601
- for_sale: String(ctx.finalInts.sell),
4602
- sell_price: {
4603
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4604
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4605
- }
4606
- };
4607
- }
4608
- updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
4609
- }
4610
- updateOperationCount++;
4611
- continue;
4612
- }
4613
- // Full rotation: old slot was virtualized in the committed working grid.
4614
- // Activate the destination slot with the existing on-chain orderId.
4615
- const slot = this.manager.orders.get(newGridId);
4616
- if (!slot) {
4617
- this.manager.logger.log(`[ROTATION] Destination slot ${newGridId} missing from master grid after COW commit - skipping activation, sync will reconcile`, 'error');
4618
- // Still clear source orderId to prevent a stale chain reference persisting
4619
- if (oldOrder?.id && oldOrder.id !== newGridId) {
4620
- const staleSource = this.manager.orders.get(oldOrder.id);
4621
- if (staleSource?.orderId) {
4622
- updatesToApply.push({
4623
- order: { ...staleSource, state: ORDER_STATES.VIRTUAL, orderId: null, rawOnChain: null },
4624
- context: 'post-rotation-source-clear'
4625
- });
4626
- }
4627
- }
4628
- continue;
4629
- }
4630
- const updatedSlot = {
4631
- ...slot,
4632
- id: newGridId,
4633
- type,
4634
- size: newSize,
4635
- price: newPrice,
4636
- state: ORDER_STATES.ACTIVE,
4637
- orderId: oldOrder?.orderId || slot.orderId || null
4638
- };
4639
- if (ctx.finalInts) {
4640
- updatedSlot.rawOnChain = {
4641
- id: updatedSlot.orderId,
4642
- for_sale: String(ctx.finalInts.sell),
4643
- sell_price: {
4644
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4645
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4646
- }
4647
- };
4648
- }
4649
- if (oldOrder && updatedSlot && this.manager.accountant) {
4650
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, updatedSlot, 'order-update', btsFeeData.updateFee, false);
4651
- }
4652
- // Ensure rotation source slot is cleared in master state.
4653
- // COW projection may keep source slots ACTIVE when target keeps a non-zero
4654
- // virtual size, but after a successful on-chain rotation the source orderId
4655
- // must no longer remain attached to the old slot.
4656
- if (oldOrder?.id && oldOrder.id !== newGridId) {
4657
- const currentSource = this.manager.orders.get(oldOrder.id);
4658
- if (currentSource && currentSource.orderId) {
4659
- updatesToApply.push({
4660
- order: {
4661
- ...currentSource,
4662
- state: ORDER_STATES.VIRTUAL,
4663
- orderId: null,
4664
- rawOnChain: null
4665
- },
4666
- context: 'post-rotation-source-clear'
4667
- });
4668
- }
4669
- }
4670
- updatesToApply.push({ order: updatedSlot, context: 'post-rotation-metadata' });
4671
- }
4672
- }
4673
- // Apply all collected updates in a single batch
4674
- if (updatesToApply.length > 0) {
4675
- await this.manager.applyGridUpdateBatch(updatesToApply.map(u => u.order), 'batch-results-process', { skipAccounting: true });
4676
- }
4677
- return {
4678
- executed: true,
4679
- hadRotation,
4680
- updateOperationCount
4681
- };
1398
+ return cowRuntime.processBatchResults(this, result, opContexts);
4682
1399
  }
4683
1400
  /**
4684
1401
  * Perform grid recalculation triggered by trigger file.
@@ -4919,89 +1636,17 @@ class DEXBot {
4919
1636
  }
4920
1637
  }
4921
1638
  async requestGridReset(reason = 'structural change', options = {}) {
4922
- if (!this.manager || typeof this._performGridResync !== 'function') {
4923
- return { skipped: true, reason: 'grid resync unavailable' };
4924
- }
4925
- const message = reason ? `[CR-RESET] ${reason}` : '[CR-RESET] grid reset requested';
4926
- this._log(`${message}; rebuilding grid from fresh on-chain state`, 'info');
4927
- const resetOptions = {
4928
- ...options,
4929
- refreshCenterPrice: options.refreshCenterPrice !== false,
4930
- };
4931
- // Skip acquire if no lock exists (no fill processing to serialize against)
4932
- // or if the caller already holds the lock (re-entrant). In the latter case
4933
- // the lock's isReentrant() check prevents queueing and runs inline.
4934
- // Otherwise, wait on the queue — this is the intended path for callers
4935
- // outside the fill-processing context that need exclusive access.
4936
- if (!this.manager._fillProcessingLock || this.manager._fillProcessingLock.isReentrant()) {
4937
- return this._performGridResync(resetOptions);
4938
- }
4939
- return this.manager._fillProcessingLock.acquire(async () => this._performGridResync(resetOptions));
1639
+ return DexbotMaintenanceRuntime.requestGridReset(this, reason, options);
4940
1640
  }
4941
1641
  _wireStructuralGridResyncRequest() {
4942
- if (!this.manager || this.manager.requestStructuralGridResync)
4943
- return;
4944
- this.manager.requestStructuralGridResync = async (reason = 'structural recovery', details = {}) => {
4945
- if (this._shuttingDown) {
4946
- return { skipped: true, reason: 'shutting down' };
4947
- }
4948
- if (this._structuralGridResyncRunning || this._structuralGridResyncTimer) {
4949
- return { skipped: true, reason: 'structural grid resync already scheduled' };
4950
- }
4951
- const unmatchedCount = Array.isArray(details?.unmatchedChainOrders)
4952
- ? details.unmatchedChainOrders.length
4953
- : 0;
4954
- this._structuralGridResyncTimer = setTimeout(async () => {
4955
- this._structuralGridResyncTimer = null;
4956
- if (this._shuttingDown)
4957
- return;
4958
- this._structuralGridResyncRunning = true;
4959
- try {
4960
- // Try the lighter persisted-grid reload before full reset.
4961
- const persistedResult = await this._recoverFromPersistedGrid();
4962
- if (persistedResult.success) {
4963
- if (this.manager?._recoveryState) {
4964
- this.manager._recoveryState = { ...this.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
4965
- }
4966
- return;
4967
- }
4968
- const suffix = unmatchedCount > 0 ? ` (${unmatchedCount} unmatched chain order(s))` : '';
4969
- this._warn(`[RECOVERY] Running structural full grid resync for ${reason}${suffix}`);
4970
- const resetResult = await this.requestGridReset('rms_structural_grid_resync', {
4971
- refreshCenterPrice: true,
4972
- });
4973
- if (resetResult && this.manager?._recoveryState) {
4974
- this.manager._recoveryState = { ...this.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
4975
- }
4976
- }
4977
- catch (err) {
4978
- this._warn(`[RECOVERY] Structural full grid resync failed: ${err.message}`);
4979
- }
4980
- finally {
4981
- this._structuralGridResyncRunning = false;
4982
- if (this.manager?._recoveryState) {
4983
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: false };
4984
- }
4985
- }
4986
- }, 0);
4987
- return { scheduled: true };
4988
- };
1642
+ return DexbotMaintenanceRuntime.wireStructuralGridResyncRequest(this);
4989
1643
  }
4990
1644
  /**
4991
1645
  * Get current metrics for monitoring and debugging.
4992
1646
  * @returns {Object} Metrics snapshot
4993
1647
  */
4994
1648
  getMetrics() {
4995
- this.manager?._cleanExpiredLocks?.();
4996
- return {
4997
- ...this._metrics,
4998
- queueDepth: this._incomingFillQueue.length,
4999
- fillProcessingLockActive: this.manager?._fillProcessingLock?.isLocked() || false,
5000
- divergenceLockActive: this.manager?._divergenceLock?.isLocked() || false,
5001
- shadowLocksActive: this.manager?.shadowOrderIds?.size || 0,
5002
- recoveryExhaustedAt: this.manager?._recoveryExhaustedAt || null,
5003
- recentFillsTracked: this._recentlyProcessedFills.size
5004
- };
1649
+ return DexbotMaintenanceRuntime.getMetrics(this);
5005
1650
  }
5006
1651
  /**
5007
1652
  * Execute grid maintenance checks in strict order with pipeline consensus.
@@ -5048,40 +1693,25 @@ class DEXBot {
5048
1693
  async _cancelDustOrders({ buy: buyDust = [], sell: sellDust = [] } = {}) {
5049
1694
  return DexbotMaintenanceRuntime.cancelDustOrders(this, { buy: buyDust, sell: sellDust });
5050
1695
  }
1696
+ /**
1697
+ * Run a single dust health check cycle: inspect grid for partials below the
1698
+ * configured dust threshold and cancel them immediately inside the fill-
1699
+ * processing lock (with timeout). Safe to call from the periodic timer or
1700
+ * once at startup.
1701
+ * @private
1702
+ */
1703
+ async _runDustHealthCheck() {
1704
+ return DexbotMaintenanceRuntime.runDustHealthCheck(this);
1705
+ }
5051
1706
  /**
5052
1707
  * Set up the periodic dust health check interval.
5053
1708
  * Catches partials below the dust threshold that were missed by the post-fill
5054
1709
  * pipeline (e.g. from a prior bot lifetime after crash/restart).
5055
- * Cancels dust immediately inside the fill-processing lock.
1710
+ * Calls _runDustHealthCheck on each tick.
5056
1711
  * @private
5057
1712
  */
5058
1713
  _setupDustHealthCheckInterval() {
5059
- const DUST_HEALTH_CHECK_INTERVAL_MS = 5 * 60 * 1000;
5060
- this._dustHealthCheckTimer = setInterval(async () => {
5061
- if (this._shuttingDown || !this.manager)
5062
- return;
5063
- try {
5064
- const health = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan?.bind(this));
5065
- const allDust = [
5066
- ...(health.buyDustOrders || []),
5067
- ...(health.sellDustOrders || []),
5068
- ];
5069
- if (allDust.length > 0) {
5070
- await this.manager._fillProcessingLock.acquire(async () => {
5071
- await this._cancelDustOrders({
5072
- buy: health.buyDustOrders,
5073
- sell: health.sellDustOrders,
5074
- });
5075
- });
5076
- }
5077
- }
5078
- catch {
5079
- // Silently retry next interval
5080
- }
5081
- }, DUST_HEALTH_CHECK_INTERVAL_MS);
5082
- if (typeof this._dustHealthCheckTimer?.unref === 'function') {
5083
- this._dustHealthCheckTimer.unref();
5084
- }
1714
+ return DexbotMaintenanceRuntime.setupDustHealthCheckInterval(this);
5085
1715
  }
5086
1716
  /**
5087
1717
  * Perform grid maintenance: fund thresholds, spread condition, grid health, divergence.