dexbot 1.3.2 → 1.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/credential-daemon.d.ts.map +1 -1
  2. package/dist/credential-daemon.js +53 -13
  3. package/dist/credential-daemon.js.map +1 -1
  4. package/dist/modules/bitshares-native/serial/types.js +1 -1
  5. package/dist/modules/bitshares-native/serial/types.js.map +1 -1
  6. package/dist/modules/chain_keys.js +1 -1
  7. package/dist/modules/chain_keys.js.map +1 -1
  8. package/dist/modules/constants.d.ts +2 -0
  9. package/dist/modules/constants.d.ts.map +1 -1
  10. package/dist/modules/constants.js +10 -0
  11. package/dist/modules/constants.js.map +1 -1
  12. package/dist/modules/credential_policy.js +1 -1
  13. package/dist/modules/credential_policy.js.map +1 -1
  14. package/dist/modules/dexbot_class.d.ts +45 -175
  15. package/dist/modules/dexbot_class.d.ts.map +1 -1
  16. package/dist/modules/dexbot_class.js +84 -3496
  17. package/dist/modules/dexbot_class.js.map +1 -1
  18. package/dist/modules/dexbot_cow_runtime.d.ts +319 -0
  19. package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -0
  20. package/dist/modules/dexbot_cow_runtime.js +1764 -0
  21. package/dist/modules/dexbot_cow_runtime.js.map +1 -0
  22. package/dist/modules/dexbot_fill_runtime.d.ts +47 -0
  23. package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
  24. package/dist/modules/dexbot_fill_runtime.js +607 -3
  25. package/dist/modules/dexbot_fill_runtime.js.map +1 -1
  26. package/dist/modules/dexbot_maintenance_runtime.d.ts +69 -0
  27. package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
  28. package/dist/modules/dexbot_maintenance_runtime.js +244 -15
  29. package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
  30. package/dist/modules/dexbot_startup_runtime.d.ts +30 -0
  31. package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -0
  32. package/dist/modules/dexbot_startup_runtime.js +539 -0
  33. package/dist/modules/dexbot_startup_runtime.js.map +1 -0
  34. package/dist/modules/dexbot_state_recovery.d.ts +128 -0
  35. package/dist/modules/dexbot_state_recovery.d.ts.map +1 -0
  36. package/dist/modules/dexbot_state_recovery.js +397 -0
  37. package/dist/modules/dexbot_state_recovery.js.map +1 -0
  38. package/dist/modules/fund_registry.d.ts.map +1 -1
  39. package/dist/modules/fund_registry.js +0 -1
  40. package/dist/modules/fund_registry.js.map +1 -1
  41. package/dist/modules/general_settings.d.ts.map +1 -1
  42. package/dist/modules/general_settings.js +0 -1
  43. package/dist/modules/general_settings.js.map +1 -1
  44. package/dist/modules/key_store.d.ts.map +1 -1
  45. package/dist/modules/key_store.js +1 -2
  46. package/dist/modules/key_store.js.map +1 -1
  47. package/dist/modules/launcher/market_adapter_runtime.d.ts.map +1 -1
  48. package/dist/modules/launcher/market_adapter_runtime.js +0 -1
  49. package/dist/modules/launcher/market_adapter_runtime.js.map +1 -1
  50. package/dist/modules/launcher/market_adapter_watchdog.d.ts.map +1 -1
  51. package/dist/modules/launcher/market_adapter_watchdog.js +1 -2
  52. package/dist/modules/launcher/market_adapter_watchdog.js.map +1 -1
  53. package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
  54. package/dist/modules/order/grid_reconcile.js +0 -1
  55. package/dist/modules/order/grid_reconcile.js.map +1 -1
  56. package/dist/modules/order/grid_reconcile_internal.js +2 -2
  57. package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
  58. package/dist/modules/order/utils/math.js +1 -1
  59. package/dist/modules/order/utils/math.js.map +1 -1
  60. package/dist/modules/order/utils/system.js +1 -1
  61. package/dist/modules/order/utils/system.js.map +1 -1
  62. package/dist/modules/order/utils/validate.d.ts.map +1 -1
  63. package/dist/modules/order/utils/validate.js +0 -1
  64. package/dist/modules/order/utils/validate.js.map +1 -1
  65. package/dist/modules/paths.d.ts +2 -0
  66. package/dist/modules/paths.d.ts.map +1 -1
  67. package/dist/modules/paths.js +13 -2
  68. package/dist/modules/paths.js.map +1 -1
  69. package/dist/scripts/verify-browser-bundle.js +25 -1
  70. package/dist/scripts/verify-browser-bundle.js.map +1 -1
  71. package/package.json +7 -2
@@ -53,33 +53,26 @@
53
53
  *
54
54
  * ===============================================================================
55
55
  */
56
- const { path } = require('./path_api');
57
- const { BitShares, waitForConnected, onReconnect: registerReconnectHook } = require('./bitshares_client');
58
- const { getStorage } = require('./storage');
59
- const storage = getStorage();
56
+ const { BitShares } = require('./bitshares_client');
60
57
  const chainKeys = require('./chain_keys');
61
- const { getKeyStore } = require('./key_store');
62
58
  const chainOrders = require('./chain_orders');
63
59
  const fundRegistry = require('./fund_registry');
64
- const { BroadcastUncertainError } = require('./dexbot_credential_client');
65
- const { OrderManager, grid: Grid } = require('./order');
66
- const { retryPersistenceIfNeeded, initializeFeeCache, } = require('./order/utils/system');
67
- const { hasExecutableActions, validateCreateTargetSlots } = require('./order/utils/validate');
68
- const { buildCreateOrderArgs, buildCreateOpFingerprint, virtualizeOrder, correctAllPriceMismatches, convertToSpreadPlaceholder, buildOutsideInPairGroups, extractBatchOperationResults, buildFillKey, formatUnmatchedChainOrder, parseChainOrder } = require('./order/utils/order');
69
- const { validateOrderSize } = require('./order/utils/math');
70
60
  const { ProcessedFillStore, PROCESSED_FILL_PERSISTENCE_MODES } = require('./order/processed_fill_store');
71
61
  const DexbotFillRuntime = require('./dexbot_fill_runtime');
72
62
  const DexbotMaintenanceRuntime = require('./dexbot_maintenance_runtime');
63
+ const DexbotStateRecovery = require('./dexbot_state_recovery');
64
+ const DexbotStartupRuntime = require('./dexbot_startup_runtime');
73
65
  const CreditRuntime = require('./credit_runtime');
74
- const { ORDER_STATES, ORDER_TYPES, REBALANCE_STATES, COW_ACTIONS, TIMING, PIPELINE_TIMING, GRID_LIMITS, MAINTENANCE, FILL_PROCESSING, DAEMON_CODES, } = require('./constants');
75
- const { PATHS, getRecalculateTriggerFile } = require('./paths');
76
- const { attemptResumePersistedGridByPriceMatch, decideStartupGridAction, reconcileGridOrders } = require('./order/grid_reconcile');
77
- const { AccountOrders } = require('./account_orders');
78
- const { parseJsonWithComments } = require('./order/utils/system');
79
- const { cloneWeightDistribution } = require('./order/utils/math');
80
- const { normalizeBotEntry } = require('./bot_settings');
66
+ const { TIMING, FILL_PROCESSING, DAEMON_CODES, } = require('./constants');
67
+ const { PATHS } = require('./paths');
81
68
  const Format = require('./order/format');
82
- const { resolveBotRuntimeSettings } = require('./runtime_settings');
69
+ const cowRuntime = require('./dexbot_cow_runtime');
70
+ function waitForConnected(...args) { return require('./bitshares_client').waitForConnected(...args); }
71
+ function getKeyStore(...args) { return require('./key_store').getKeyStore(...args); }
72
+ function hasExecutableActions(...args) { return require('./order/utils/validate').hasExecutableActions(...args); }
73
+ function getRecalculateTriggerFile(...args) { return require('./paths').getRecalculateTriggerFile(...args); }
74
+ function cloneWeightDistribution(...args) { return require('./order/utils/math').cloneWeightDistribution(...args); }
75
+ function resolveBotRuntimeSettings(...args) { return require('./runtime_settings').resolveBotRuntimeSettings(...args); }
83
76
  const PROFILES_BOTS_FILE = PATHS.PROFILES.BOTS_JSON;
84
77
  const PROFILES_DIR = PATHS.PROFILES_DIR;
85
78
  class DEXBot {
@@ -324,21 +317,7 @@ class DEXBot {
324
317
  * @private
325
318
  */
326
319
  async _persistAndRecoverIfNeeded() {
327
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
328
- const validation = await this.manager.persistGrid();
329
- if (!validation.isValid) {
330
- this._warn(`Startup validation failed: ${validation.reason}. Triggering immediate recovery...`);
331
- // Trigger centralized recovery (Hard Reset)
332
- const recoveryValidation = await this.manager.accountant._performStateRecovery(this.manager);
333
- if (recoveryValidation.isValid) {
334
- this._log(`✓ Startup recovery successful. Persistent state restored.`);
335
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
336
- await this.manager.persistGrid();
337
- }
338
- else {
339
- this._warn(`Startup recovery failed: ${recoveryValidation.reason}. Bot proceeding with caution.`);
340
- }
341
- }
320
+ return DexbotStateRecovery.persistAndRecoverIfNeeded(this);
342
321
  }
343
322
  /**
344
323
  * Snapshot the recently queued fill keys as a plain object for crash-durable persistence.
@@ -348,42 +327,14 @@ class DEXBot {
348
327
  * @returns {Record<string, number>}
349
328
  */
350
329
  _getRecentFillKeysSnapshot() {
351
- const snapshot = {};
352
- const now = Date.now();
353
- // 1. Collect live keys from the in-memory map
354
- for (const [key, timestamp] of this._recentlyQueuedFills) {
355
- if (now - Number(timestamp) < this._fillDedupeWindowMs) {
356
- snapshot[key] = Number(timestamp);
357
- }
358
- }
359
- // 2. Merge with the previous snapshot — carry forward any keys that
360
- // were present in a prior persist cycle, are still within the
361
- // dedup window, but have been evicted from _recentlyQueuedFills
362
- // (e.g. by TTL-based pruning between two persistGrid calls).
363
- // This prevents a crash + restart from losing keys that were
364
- // processed but not yet persisted.
365
- if (this.manager?._recentFillKeysSnapshot) {
366
- for (const [key, timestamp] of Object.entries(this.manager._recentFillKeysSnapshot)) {
367
- if (!(key in snapshot) && now - Number(timestamp) < this._fillDedupeWindowMs) {
368
- snapshot[key] = Number(timestamp);
369
- }
370
- }
371
- }
372
- return snapshot;
330
+ return DexbotStateRecovery.getRecentFillKeysSnapshot(this);
373
331
  }
374
332
  /**
375
333
  * Get current pipeline signal state for congestion checks.
376
334
  * @returns {{incomingFillQueueLength: number, shadowLocks: number, batchInFlight: boolean, recoveryInFlight: boolean, broadcasting: boolean}}
377
335
  */
378
336
  _getPipelineSignals() {
379
- this.manager?._cleanExpiredLocks?.();
380
- return {
381
- incomingFillQueueLength: this._incomingFillQueue.length,
382
- shadowLocks: this.manager?.shadowOrderIds?.size || 0,
383
- batchInFlight: this._batchInFlight,
384
- recoveryInFlight: this._recoverySyncInFlight,
385
- broadcasting: this.manager?.isBroadcastingActive?.() || false
386
- };
337
+ return DexbotMaintenanceRuntime.getPipelineSignals(this);
387
338
  }
388
339
  /**
389
340
  * Mark that grid activity occurred (updates idle timer).
@@ -391,8 +342,7 @@ class DEXBot {
391
342
  * @returns {void}
392
343
  */
393
344
  _markGridActivity(reason = 'activity') {
394
- this._lastGridActivityAt = Date.now();
395
- this.manager?.logger?.log?.(`[MAINT-IDLE] Activity observed: ${reason}`, 'debug');
345
+ return DexbotMaintenanceRuntime.markGridActivity(this, reason);
396
346
  }
397
347
  /**
398
348
  * Trigger a full state recovery sync (fetch chain + sync from open orders + persist).
@@ -400,25 +350,7 @@ class DEXBot {
400
350
  * @returns {Promise<void>}
401
351
  */
402
352
  async _triggerStateRecoverySync(reason = 'state recovery sync') {
403
- if (!this.manager)
404
- return;
405
- if (this._recoverySyncInFlight) {
406
- this.manager.logger.log(`[RECOVERY] Skipping duplicate recovery request: ${reason}`, 'warn');
407
- return;
408
- }
409
- this._recoverySyncInFlight = true;
410
- try {
411
- this.manager.logger.log(`Triggering state recovery sync (${reason})...`, 'info');
412
- await this.manager.fetchAccountTotals(this.accountId);
413
- const openOrders = await chainOrders.readOpenOrders(this.accountId);
414
- await this.manager.syncFromOpenOrders(openOrders, { skipAccounting: true });
415
- if (typeof this.manager.persistGrid === 'function') {
416
- await this.manager.persistGrid();
417
- }
418
- }
419
- finally {
420
- this._recoverySyncInFlight = false;
421
- }
353
+ return DexbotStateRecovery.triggerStateRecoverySync(this, reason);
422
354
  }
423
355
  /**
424
356
  * Abort the current flow if an illegal state signal was raised.
@@ -426,14 +358,7 @@ class DEXBot {
426
358
  * @returns {Promise<boolean>} True if flow was aborted
427
359
  */
428
360
  async _abortFlowIfIllegalState(flowContext) {
429
- const illegalSignal = this.manager?.consumeIllegalStateSignal?.();
430
- if (!illegalSignal) {
431
- return false;
432
- }
433
- this.manager.logger.log(`[HARD-ABORT] ${flowContext} aborted due to illegal state (${illegalSignal.context}): ${illegalSignal.message}`, 'error');
434
- await this._triggerStateRecoverySync(`hard-abort ${flowContext}`);
435
- this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
436
- return true;
361
+ return DexbotStateRecovery.abortFlowIfIllegalState(this, flowContext);
437
362
  }
438
363
  /**
439
364
  * Handle a hard abort from batch processing due to illegal state or accounting failure.
@@ -443,24 +368,7 @@ class DEXBot {
443
368
  * @returns {Promise<Object>} Abort result object
444
369
  */
445
370
  async _handleBatchHardAbort(err, phase = 'batch processing', opsCount = 0) {
446
- const baseResult = { executed: false, hadRotation: false };
447
- const opsInfo = opsCount > 0 ? ` with ${opsCount} ops` : '';
448
- if (err?.code === 'ILLEGAL_ORDER_STATE') {
449
- const illegalSignal = this.manager.consumeIllegalStateSignal?.();
450
- await this._triggerStateRecoverySync(illegalSignal?.message || `illegal order state during ${phase}${opsInfo}`);
451
- this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
452
- return { ...baseResult, abortedForIllegalState: true };
453
- }
454
- if (err?.code === 'ACCOUNTING_COMMITMENT_FAILED') {
455
- const accountingSignal = this.manager.consumeAccountingFailureSignal?.();
456
- const reason = accountingSignal
457
- ? `accounting lock failure (${accountingSignal.side} ${Format.formatAmount8(accountingSignal.amount)}) during ${accountingSignal.context}`
458
- : `accounting commitment lock failure during ${phase}${opsInfo}`;
459
- await this._triggerStateRecoverySync(reason);
460
- this._maintenanceCooldownCycles = Math.max(this._maintenanceCooldownCycles, 1);
461
- return { ...baseResult, abortedForAccountingFailure: true };
462
- }
463
- return null;
371
+ return DexbotStateRecovery.handleBatchHardAbort(this, err, phase, opsCount);
464
372
  }
465
373
  /**
466
374
  * Apply recoverable grid updates (order virtualisation) after a batch failure.
@@ -469,30 +377,7 @@ class DEXBot {
469
377
  * @returns {Promise<number>} Number of updates applied
470
378
  */
471
379
  async _applyRecoverableGridUpdates(updates, context = 'recoverable-grid-update') {
472
- if (!this.manager || !Array.isArray(updates) || updates.length === 0) {
473
- return 0;
474
- }
475
- let applied;
476
- if (typeof this.manager.applyGridUpdateBatch === 'function') {
477
- await this.manager.applyGridUpdateBatch(updates, context);
478
- applied = updates.length;
479
- }
480
- else {
481
- applied = 0;
482
- for (const update of updates) {
483
- if (typeof this.manager._updateOrder !== 'function')
484
- break;
485
- await this.manager._updateOrder(update, context);
486
- applied++;
487
- }
488
- }
489
- // Persist master grid mutations applied outside COW (stale-order
490
- // virtualization, size-drift corrections). These run in the COW catch
491
- // handler where the success-path persistGrid is never reached.
492
- if (applied > 0 && typeof this.manager.persistGrid === 'function') {
493
- await this.manager.persistGrid();
494
- }
495
- return applied;
380
+ return DexbotStateRecovery.applyRecoverableGridUpdates(this, updates, context);
496
381
  }
497
382
  /**
498
383
  * Recover from explicit stale order errors by virtualizing affected grid slots.
@@ -501,36 +386,7 @@ class DEXBot {
501
386
  * @returns {Promise<{executed: boolean, hadRotation: boolean, stale: boolean, recoveredByVirtualization?: boolean}>}
502
387
  */
503
388
  async _recoverExplicitStaleOrders(staleOrderIds, reason = 'stale order cleanup') {
504
- const staleIds = Array.from(staleOrderIds || []).filter(Boolean);
505
- if (staleIds.length === 0) {
506
- return { executed: false, hadRotation: false, stale: false };
507
- }
508
- this.manager.logger.log(`[COW] Stale order(s) detected: ${staleIds.join(', ')}. Applying targeted cleanup.`, 'warn');
509
- const updates = [];
510
- for (const [, gridOrder] of this.manager.orders.entries()) {
511
- if (!gridOrder?.orderId || !staleOrderIds.has(gridOrder.orderId))
512
- continue;
513
- this._staleCleanedOrderIds.set(gridOrder.orderId, Date.now());
514
- updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
515
- }
516
- // Register any stale IDs that had no matching grid slot
517
- for (const orderId of staleIds) {
518
- if (!this._staleCleanedOrderIds.has(orderId)) {
519
- this._staleCleanedOrderIds.set(orderId, Date.now());
520
- }
521
- }
522
- if (updates.length > 0) {
523
- await this._applyRecoverableGridUpdates(updates, reason);
524
- }
525
- else {
526
- this.manager.logger.log(`[COW] No local grid slot matched stale order cleanup request (${staleIds.join(', ')}).`, 'debug');
527
- }
528
- return {
529
- executed: false,
530
- hadRotation: false,
531
- stale: true,
532
- recoveredByVirtualization: updates.length > 0
533
- };
389
+ return DexbotStateRecovery.recoverExplicitStaleOrders(this, staleOrderIds, reason);
534
390
  }
535
391
  /**
536
392
  * Recover from on-chain size drift detected during batch broadcast.
@@ -538,32 +394,7 @@ class DEXBot {
538
394
  * @returns {Promise<{executed: boolean, hadRotation: boolean, recoveredBySync: boolean, reason: string}>}
539
395
  */
540
396
  async _recoverBatchSizeDrift(err, opContexts = []) {
541
- // Try a targeted fix first: extract the affected order IDs from the
542
- // operation contexts and correct them directly from chain. This
543
- // avoids a full state recovery sync in the common single-order case.
544
- const affectedOrderIds = this._extractSizeDriftOrderIds(opContexts);
545
- if (affectedOrderIds.length > 0) {
546
- this.manager.logger.log(`[COW] Targeted size-drift repair for ${affectedOrderIds.length} order(s): ${affectedOrderIds.join(', ')}`, 'debug');
547
- const repaired = await this._targetedOrderRepair(affectedOrderIds);
548
- if (repaired) {
549
- return {
550
- executed: false,
551
- hadRotation: false,
552
- recoveredBySync: true,
553
- reason: 'ORDER_SIZE_DRIFT_TARGETED'
554
- };
555
- }
556
- this.manager.logger.log('[COW] Targeted repair failed, falling back to full state recovery sync.', 'warn');
557
- }
558
- const reason = `recoverable size drift during COW batch: ${err.message}`;
559
- this.manager.logger.log(`[COW] Recovering from on-chain size drift via recovery sync: ${err.message}`, 'warn');
560
- await this._triggerStateRecoverySync(reason);
561
- return {
562
- executed: false,
563
- hadRotation: false,
564
- recoveredBySync: true,
565
- reason: 'ORDER_SIZE_DRIFT'
566
- };
397
+ return DexbotStateRecovery.recoverBatchSizeDrift(this, err, opContexts);
567
398
  }
568
399
  /**
569
400
  * Extract chain order IDs from opContexts for operations that could
@@ -572,18 +403,7 @@ class DEXBot {
572
403
  * @returns {string[]} Unique chain order IDs
573
404
  */
574
405
  _extractSizeDriftOrderIds(opContexts) {
575
- if (!Array.isArray(opContexts))
576
- return [];
577
- const ids = new Set();
578
- for (const ctx of opContexts) {
579
- if (ctx?.kind === 'size-update' && ctx?.updateInfo?.partialOrder?.orderId) {
580
- ids.add(ctx.updateInfo.partialOrder.orderId);
581
- }
582
- else if (ctx?.kind === 'rotation' && ctx?.rotation?.oldOrder?.orderId) {
583
- ids.add(ctx.rotation.oldOrder.orderId);
584
- }
585
- }
586
- return Array.from(ids);
406
+ return DexbotStateRecovery.extractSizeDriftOrderIds(opContexts);
587
407
  }
588
408
  /**
589
409
  * Reload the entire grid from the persisted on-disk snapshot and reconcile
@@ -596,87 +416,7 @@ class DEXBot {
596
416
  * @returns {Promise<{success: boolean, reason?: string}>}
597
417
  */
598
418
  async _recoverFromPersistedGrid() {
599
- if (!this.accountOrders || !this.manager) {
600
- return { success: false, reason: 'accountOrders or manager unavailable' };
601
- }
602
- const accountRef = this.accountId || this.account?.id || this.account;
603
- if (!accountRef) {
604
- return { success: false, reason: 'no account reference' };
605
- }
606
- this.manager.logger.log('[RECOVERY] Attempting full grid reload from persisted snapshot...', 'warn');
607
- try {
608
- // 1. Force reload from disk
609
- const persistedGrid = this.accountOrders.loadGrid(true);
610
- if (!persistedGrid || persistedGrid.length === 0) {
611
- return { success: false, reason: 'no persisted grid on disk' };
612
- }
613
- const boundaryIdx = this.accountOrders.loadBoundaryIdx(true);
614
- // 2. Load into manager (same path as startup at line 1340)
615
- await Grid.loadGrid(this.manager, persistedGrid, boundaryIdx);
616
- // Gap 1: Grid snapshot sanity check — shared logic with startup path.
617
- if (await this._rejectCorruptedGridSnapshot('recovery')) {
618
- return { success: false, reason: 'corrupted grid snapshot rejected (fund drift)' };
619
- }
620
- // 3. Read current chain state
621
- const chainOpenOrders = await chainOrders.readOpenOrders(accountRef);
622
- // 4. Reconcile
623
- if (chainOpenOrders.length > 0 && this.manager?.syncFromOpenOrders) {
624
- await this.manager.syncFromOpenOrders(chainOpenOrders, {
625
- skipAccounting: true,
626
- });
627
- }
628
- // 5. Persist the reconciled state
629
- if (typeof this.manager.persistGrid === 'function') {
630
- await this.manager.persistGrid();
631
- }
632
- const assets = this.manager?.assets;
633
- const matchedCount = assets
634
- ? chainOpenOrders.filter(o => parseChainOrder(o, assets) !== null).length
635
- : chainOpenOrders.length;
636
- this.manager.logger.log(`[RECOVERY] Grid reloaded from persisted snapshot: ${this.manager.orders.size} orders, ` +
637
- `${matchedCount} on-chain orders synced`, 'info');
638
- // Gap 2: Check for unmatched chain orders after reload + sync.
639
- // If the sync resolved all unmatched entries, _lastUnmatchedChainOrders
640
- // was cleared by the sync engine. If any remain, the persisted snapshot
641
- // produced an inconsistent grid — reject so the structural resync
642
- // falls through to requestGridReset (full rebuild from chain).
643
- const remainingUnmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
644
- ? this.manager._lastUnmatchedChainOrders
645
- : [];
646
- if (remainingUnmatched.length > 0) {
647
- const sample = remainingUnmatched.slice(0, 3)
648
- .map(o => this._formatUnmatchedChainOrderForLog(o))
649
- .join(' | ');
650
- this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but ${remainingUnmatched.length} unmatched chain order(s) ` +
651
- `remain${sample ? ` (${sample})` : ''}. Rejecting — full grid reset required.`, 'warn');
652
- return { success: false, reason: `grid inconsistent after reload: ${remainingUnmatched.length} unmatched remain` };
653
- }
654
- // If the reloaded grid is still bloated, reject recovery so the
655
- // caller falls through to a full grid reset (requestGridReset).
656
- // Without this check, a bloated snapshot gets accepted as "success"
657
- // and the structural-resync loop loads the same broken state forever.
658
- //
659
- // NOTE: loadGrid() already fires requestStructuralGridResync when it
660
- // detects bloat internally, so the inner async resync may be in flight
661
- // by the time this outer check runs. That's fine — the structural-resync
662
- // gate (_structuralGridResyncRunning / _structuralGridResyncTimer) dedup's
663
- // concurrent requests. This outer check exists so the synchronous return
664
- // value is honest about the state; the inner resync is a safety net.
665
- const { isGridBloated } = require('./order/grid');
666
- const ordersArr = Array.from(this.manager.orders.values());
667
- const bloatPostRecovery = isGridBloated(this.manager, ordersArr);
668
- if (bloatPostRecovery.bloated) {
669
- const d = bloatPostRecovery.details;
670
- this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but still bloated ` +
671
- `(${d.gridSize} slots, max ${d.maxAllowed}). Rejecting — full grid reset required.`, 'warn');
672
- return { success: false, reason: 'grid still bloated after reload' };
673
- }
674
- return { success: true };
675
- }
676
- catch (err) {
677
- this.manager.logger.log(`[RECOVERY] Full grid reload from persisted snapshot failed: ${err.message}`, 'error');
678
- return { success: false, reason: err.message };
679
- }
419
+ return DexbotStateRecovery.recoverFromPersistedGrid(this);
680
420
  }
681
421
  /**
682
422
  * Reject a corrupted grid snapshot when catastrophic fund drift is detected.
@@ -687,25 +427,7 @@ class DEXBot {
687
427
  * @returns {Promise<boolean>} True if the snapshot was rejected (cleared).
688
428
  */
689
429
  async _rejectCorruptedGridSnapshot(context) {
690
- if (!this.manager?.checkFundDriftAfterFills)
691
- return false;
692
- const driftCheck = this.manager.checkFundDriftAfterFills();
693
- if (driftCheck.isValid)
694
- return false;
695
- const tag = context === 'recovery' ? '[RECOVERY][SNAPSHOT-REJECT]' : '[SNAPSHOT-REJECT]';
696
- this._warn(`${tag} Corrupted grid snapshot detected: ` +
697
- `drift sell=${driftCheck.driftSell.toFixed(2)} buy=${driftCheck.driftBuy.toFixed(2)}. ` +
698
- `Deleting corrupted snapshot.`);
699
- if (this.accountOrders && typeof this.accountOrders.clearGrid === 'function') {
700
- try {
701
- await this.accountOrders.clearGrid();
702
- this._warn(`${tag} Corrupted grid snapshot deleted.`);
703
- }
704
- catch (clearErr) {
705
- this._warn(`${tag} Failed to delete corrupted snapshot: ${clearErr.message}`);
706
- }
707
- }
708
- return true;
430
+ return DexbotStateRecovery.rejectCorruptedGridSnapshot(this, context);
709
431
  }
710
432
  /**
711
433
  * Attempt to repair size-drift for specific order IDs by reading their
@@ -715,141 +437,16 @@ class DEXBot {
715
437
  * @returns {Promise<boolean>} True if all affected orders were repaired
716
438
  */
717
439
  async _targetedOrderRepair(orderIds) {
718
- try {
719
- const objects = await BitShares.db.get_objects(orderIds);
720
- if (!Array.isArray(objects) || objects.length !== orderIds.length)
721
- return false;
722
- const updates = [];
723
- for (let i = 0; i < orderIds.length; i++) {
724
- const chainOrder = objects[i];
725
- const gridOrder = Array.from(this.manager.orders.values())
726
- .find((o) => o.orderId === orderIds[i]);
727
- if (!gridOrder)
728
- continue;
729
- if (!chainOrder || typeof chainOrder.for_sale === 'undefined') {
730
- // Order no longer exists on chain -> fully filled or cancelled.
731
- updates.push({ ...virtualizeOrder(gridOrder), size: 0 });
732
- }
733
- else {
734
- const chainUnits = Number(chainOrder.for_sale);
735
- if (Number.isFinite(chainUnits)) {
736
- const { blockchainToFloat } = require('./order/utils/math');
737
- const prec = gridOrder.type === ORDER_TYPES.SELL
738
- ? this.manager.assets.assetA.precision
739
- : this.manager.assets.assetB.precision;
740
- const floatSize = blockchainToFloat(chainUnits, prec);
741
- if (floatSize !== gridOrder.size) {
742
- updates.push({
743
- id: gridOrder.id,
744
- size: floatSize,
745
- rawOnChain: chainOrder,
746
- });
747
- }
748
- }
749
- }
750
- }
751
- if (updates.length > 0) {
752
- await this._applyRecoverableGridUpdates(updates, 'targeted-size-drift-repair');
753
- }
754
- return true;
755
- }
756
- catch (err) {
757
- this.manager.logger.log(`[COW] Targeted order repair failed: ${err.message}`, 'debug');
758
- return false;
759
- }
440
+ return DexbotStateRecovery.targetedOrderRepair(this, orderIds);
760
441
  }
761
442
  /**
762
443
  * Initialize bot state from storage and blockchain.
763
444
  * Consolidates common initialization logic for start() and startWithPrivateKey().
764
- * @returns {{persistedGrid: Object, persistedBtsFeesOwed: number, persistedBoundaryIdx: number, persistedBtsBalance: number}}
445
+ * @returns {{persistedGrid: Object, persistedBtsFeesOwed: number, persistedBoundaryIdx: number, persistedBtsBalance: number, persistedRecentFillKeys: Object}}
765
446
  * @private
766
447
  */
767
448
  async _initializeStartupState() {
768
- // Create AccountOrders with bot-specific file (one file per bot)
769
- this.accountOrders = new AccountOrders({ botKey: this.config.botKey });
770
- this._processedFillStore.configure({
771
- accountOrders: this.accountOrders
772
- });
773
- // Load persisted processed fills to prevent reprocessing after restart
774
- const loadedPersistedFills = this._processedFillStore.loadPersisted({
775
- minTimestamp: Date.now() - this._fillRecordRetentionMs
776
- });
777
- if (loadedPersistedFills > 0) {
778
- this._log(`Loaded ${loadedPersistedFills} persisted fill records to prevent reprocessing`);
779
- }
780
- // Ensure bot metadata is properly initialized in storage BEFORE any Grid operations
781
- const raw = storage.readFile(PROFILES_BOTS_FILE);
782
- const allBotsConfig = parseJsonWithComments(raw).bots || [];
783
- const myBotConfig = allBotsConfig
784
- .map((b, originalIdx) => b.active !== false ? normalizeBotEntry(b, originalIdx) : null)
785
- .find(b => b && b.botKey === this.config.botKey);
786
- if (myBotConfig) {
787
- await this.accountOrders.syncMeta(myBotConfig);
788
- }
789
- if (!this.manager) {
790
- const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
791
- this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
792
- this.manager.account = this.account;
793
- this.manager.accountId = this.accountId;
794
- this.manager.accountOrders = this.accountOrders;
795
- }
796
- this._wireStructuralGridResyncRequest();
797
- this._wireProcessedFillTracking();
798
- this.manager.startBootstrap();
799
- try {
800
- // Fetch account totals from blockchain at startup to initialize funds
801
- try {
802
- if (this.accountId && this.config.assetA && this.config.assetB) {
803
- await this.manager._initializeAssets();
804
- await this.manager.fetchAccountTotals(this.accountId);
805
- this._log('Fetched blockchain account balances at startup');
806
- }
807
- }
808
- catch (err) {
809
- this._log(`Startup balance fetch FAILED: ${err.message}. Order sizing may be incorrect until next successful sync.`, 'error');
810
- }
811
- // Ensure fee cache is initialized before any fill processing that calls getAssetFees().
812
- try {
813
- await initializeFeeCache([this.config || {}], BitShares);
814
- }
815
- catch (err) {
816
- this._log(`Fee cache initialization FAILED: ${err.message}. Fee calculations will use defaults until cache is refreshed.`, 'error');
817
- }
818
- const persistedGrid = this.accountOrders.loadGrid();
819
- // CRITICAL REPAIR: Strip fake orderIds where orderId === id (e.g. "slot-0")
820
- let repairedGrid = persistedGrid;
821
- if (persistedGrid && persistedGrid.length > 0) {
822
- let repairCount = 0;
823
- repairedGrid = persistedGrid.map(order => {
824
- if (order && order.orderId && order.orderId === order.id) {
825
- repairCount++;
826
- const repairedOrder = { ...order, orderId: '' };
827
- if (repairedOrder.state === ORDER_STATES.ACTIVE || repairedOrder.state === ORDER_STATES.PARTIAL) {
828
- repairedOrder.state = ORDER_STATES.VIRTUAL;
829
- }
830
- return repairedOrder;
831
- }
832
- return order;
833
- });
834
- if (repairCount > 0) {
835
- this._log(`[REPAIR] Stripped ${repairCount} fake orderId(s) from persisted grid to restore rebalancing logic.`);
836
- }
837
- }
838
- const persistedBtsFeesOwed = this.accountOrders.loadBtsFeesOwed();
839
- const persistedBoundaryIdx = this.accountOrders.loadBoundaryIdx();
840
- const persistedBtsBalance = this.accountOrders.loadBtsBalance();
841
- const persistedRecentFillKeys = this.accountOrders.loadRecentFillKeys();
842
- return {
843
- persistedGrid: repairedGrid,
844
- persistedBtsFeesOwed,
845
- persistedBoundaryIdx,
846
- persistedBtsBalance,
847
- persistedRecentFillKeys,
848
- };
849
- }
850
- finally {
851
- this.manager.finishBootstrap();
852
- }
449
+ return DexbotStartupRuntime.initializeStartupState(this);
853
450
  }
854
451
  /**
855
452
  * Wire processed fill tracking into the manager.
@@ -995,458 +592,7 @@ class DEXBot {
995
592
  * @private
996
593
  */
997
594
  async _finishStartupSequence(startupState) {
998
- let { persistedGrid, persistedBtsFeesOwed, persistedBoundaryIdx, persistedBtsBalance, persistedRecentFillKeys, } = startupState;
999
- try {
1000
- // CRITICAL: Activate fill listener EARLY - before ANY operations that place orders
1001
- // This ensures fills during trigger reset and grid initialization are captured
1002
- if (typeof this._fillsUnsubscribe === 'function') {
1003
- await this._fillsUnsubscribe().catch(() => { });
1004
- }
1005
- this._fillsUnsubscribe = await chainOrders.listenForFills(this.account || undefined, this._createFillCallback(chainOrders));
1006
- if (typeof this._fillsUnsubscribe !== 'function') {
1007
- this._warn('Fill listener did not provide an unsubscribe handler. Shutdown cleanup may be incomplete.');
1008
- this._fillsUnsubscribe = null;
1009
- }
1010
- this._log('Fill listener activated (ready to process fills during startup)');
1011
- // Register reconnection callback for safety-net sync after websocket reconnect
1012
- if (!this._reconnectUnregister) {
1013
- this._reconnectUnregister = registerReconnectHook(() => {
1014
- this._log('Blockchain connection re-established; scheduling safety-net sync');
1015
- const runSafetyNetSync = async () => {
1016
- if (this.manager && this.accountId && !this._shuttingDown && !this.config.dryRun) {
1017
- // Cap the entire safety-net sync at TIMING.SAFETY_NET_SYNC_TIMEOUT_MS so it can
1018
- // never hold _fillProcessingLock longer than the
1019
- // 20s shutdown lock timeout. readOpenOrders +
1020
- // synchronizeWithChain + batch + persist can be
1021
- // ~45s worst case without a cap, which would
1022
- // stall shutdown until the 20s timeout fires.
1023
- const safetyNetTimeoutMs = this.config.timing?.SAFETY_NET_SYNC_TIMEOUT_MS;
1024
- let safetyNetTimer;
1025
- const workPromise = this.manager._fillProcessingLock.acquire(async () => {
1026
- if (this._shuttingDown)
1027
- return;
1028
- const chainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
1029
- if (this._shuttingDown)
1030
- return;
1031
- const syncResult = await this.manager.synchronizeWithChain(chainOpenOrders, 'readOpenOrders');
1032
- if (this._shuttingDown)
1033
- return;
1034
- if (syncResult?.filledOrders?.length > 0) {
1035
- this._refreshDynamicWeightDistribution('post-reconnect sync fill');
1036
- this._log(`Post-reconnect sync: ${syncResult.filledOrders.length} grid order(s) found filled.`, 'info');
1037
- await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'post-reconnect sync fill');
1038
- if (this._shuttingDown)
1039
- return;
1040
- }
1041
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
1042
- await this.manager.persistGrid();
1043
- // Cancel any dust created by reconnect fills immediately.
1044
- if (!this._shuttingDown) {
1045
- try {
1046
- const reconnectHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
1047
- await this._cancelDustOrders({
1048
- buy: reconnectHealth.buyDustOrders,
1049
- sell: reconnectHealth.sellDustOrders,
1050
- });
1051
- }
1052
- catch (_dustErr) {
1053
- this._warn(`[RECONNECT] Dust cancel failed: ${_dustErr.message}`);
1054
- }
1055
- }
1056
- });
1057
- try {
1058
- await Promise.race([
1059
- workPromise,
1060
- new Promise((_, reject) => {
1061
- safetyNetTimer = setTimeout(() => reject(new Error(`Safety-net sync exceeded ${safetyNetTimeoutMs}ms cap`)), safetyNetTimeoutMs);
1062
- })
1063
- ]);
1064
- }
1065
- catch (capErr) {
1066
- const fallback = await Promise.race([
1067
- workPromise.then(() => ({ ok: true })),
1068
- new Promise(resolve => setTimeout(() => resolve({ ok: false }), 0))
1069
- ]);
1070
- if (fallback.ok) {
1071
- this._log(`Safety-net sync completed despite timeout — ignoring spurious error.`, 'info');
1072
- }
1073
- else {
1074
- this._warn(`Post-reconnect safety-net sync aborted: ${capErr?.message || capErr}`);
1075
- }
1076
- }
1077
- finally {
1078
- if (safetyNetTimer)
1079
- clearTimeout(safetyNetTimer);
1080
- }
1081
- }
1082
- };
1083
- // The setImmediate callback returns a Promise; we MUST attach
1084
- // a .catch so a rejection here does not propagate to
1085
- // process.on('unhandledRejection') and tear down the bot.
1086
- setImmediate(() => {
1087
- runSafetyNetSync().catch(err => {
1088
- try {
1089
- this._warn('Post-reconnect safety-net sync failed: ' + (err?.message || err));
1090
- }
1091
- catch (_warnErr) {
1092
- // _warn itself inaccessible — last line of defense.
1093
- }
1094
- });
1095
- });
1096
- });
1097
- }
1098
- // CRITICAL: Handle any pending trigger file reset FIRST before any other startup operations
1099
- const hadTriggerReset = await this._handlePendingTriggerReset();
1100
- // CRITICAL: After trigger reset, skip normal startup - grid is already fully initialized
1101
- // The trigger reset already did: grid init, order placement, sync, and persistence
1102
- if (hadTriggerReset) {
1103
- this._log('Trigger reset completed. Skipping normal startup grid initialization.');
1104
- // Post-bootstrap validation and fill processing
1105
- await this.manager._fillProcessingLock.acquire(async () => {
1106
- // STEP 1: Check for fills that occurred during trigger reset
1107
- // These are orders that got filled while Grid.recalculateGrid() was running.
1108
- // The filled slots need new orders placed on them.
1109
- if (this._incomingFillQueue.length > 0) {
1110
- this._log(`[POST-RESET] ${this._incomingFillQueue.length} fill(s) detected during trigger reset. Processing...`);
1111
- // Process fills - this will place new orders on the filled slots
1112
- // Use normal fill processing since bootstrap is complete
1113
- const fills = this._incomingFillQueue.splice(0);
1114
- const processedFillKeys = new Set();
1115
- let requiresOpenOrdersSync = false;
1116
- for (const fill of fills) {
1117
- if (!fill || fill.op?.[0] !== 4)
1118
- continue;
1119
- const fillOp = fill.op[1];
1120
- const gridOrder = this.manager.orders.get(fillOp.order_id) ||
1121
- Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
1122
- if (!gridOrder) {
1123
- // CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
1124
- // This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
1125
- let orphanFillKey = buildFillKey(fill);
1126
- if (!orphanFillKey) {
1127
- orphanFillKey = this._buildOrphanFillFallbackKey(fill);
1128
- }
1129
- if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[POST-RESET]', fillOp.order_id)) {
1130
- continue;
1131
- }
1132
- this._log(`[POST-RESET] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
1133
- const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
1134
- context: 'POST-RESET',
1135
- logger: { log: this._log.bind(this) },
1136
- replayMessage: (op) => `[POST-RESET] Replay detected for orphan fill ${op.order_id}; skipping duplicate credit`
1137
- });
1138
- if (accountingResult.status === 'missing_key') {
1139
- requiresOpenOrdersSync = true;
1140
- }
1141
- continue;
1142
- }
1143
- this._log(`[POST-RESET] Processing fill for ${gridOrder.type} order ${gridOrder.id} at price ${gridOrder.price}`);
1144
- const trackedFillKey = buildFillKey(fill);
1145
- if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[POST-RESET]', fillOp.order_id)) {
1146
- continue;
1147
- }
1148
- this.manager.lockOrders([gridOrder.id]);
1149
- try {
1150
- const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
1151
- context: 'POST-RESET',
1152
- logger: { log: this._log.bind(this) },
1153
- replayMessage: (op) => `[POST-RESET] Replay detected for ${op.order_id}; skipping duplicate rebalance`
1154
- });
1155
- if (accountingResult.status === 'missing_key') {
1156
- requiresOpenOrdersSync = true;
1157
- continue;
1158
- }
1159
- if (accountingResult.status !== 'applied') {
1160
- continue;
1161
- }
1162
- // Process this fill through the full rebalance pipeline
1163
- // This will shift the boundary and place a new order on the filled slot
1164
- const result = await this._processFillsWithBatching([gridOrder], new Set(), `[POST-RESET] fill ${gridOrder.id}`);
1165
- if (result.aborted) {
1166
- this._warn('[POST-RESET] Aborted batch due to illegal state; skipping grid persistence this cycle');
1167
- continue;
1168
- }
1169
- }
1170
- finally {
1171
- this.manager.unlockOrders([gridOrder.id]);
1172
- }
1173
- }
1174
- if (requiresOpenOrdersSync) {
1175
- this._log('[POST-RESET] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
1176
- const postResetChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
1177
- const syncResult = await this.manager.syncFromOpenOrders(postResetChainOpenOrders);
1178
- if (syncResult.filledOrders?.length > 0) {
1179
- await this._processFillsWithBatching(syncResult.filledOrders, new Set(), '[POST-RESET] open-orders fallback');
1180
- }
1181
- }
1182
- await this._flushProcessedFillPersistence('post-reset-batch');
1183
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
1184
- await this.manager.persistGrid();
1185
- }
1186
- // STEP 2: Refresh chain truth before spread correction. Trigger
1187
- // reset can create/cancel orders and fills can arrive while the
1188
- // reset is running; spread decisions must not use stale local grid.
1189
- const { aborted: postResetAborted, hasUnmatched: postResetUnmatched } = await this._syncOpenOrdersAndProcessFills('[POST-RESET] pre-spread');
1190
- if (postResetUnmatched) {
1191
- this._warn(`[POST-RESET] Skipping spread correction: ${postResetUnmatched} unmatched chain order(s) require maintenance reconciliation`);
1192
- }
1193
- // STEP 3: Spread check AFTER fills are processed and chain truth refreshed
1194
- await this.manager.recalculateFunds();
1195
- if (!postResetAborted && !postResetUnmatched) {
1196
- const spreadResult = await this.manager.checkSpreadCondition(BitShares, this.updateOrdersOnChainPlan.bind(this));
1197
- if (spreadResult && spreadResult.ordersPlaced > 0) {
1198
- this._log(`✓ Spread correction after trigger reset: ${spreadResult.ordersPlaced} order(s) placed`);
1199
- await this._persistAndRecoverIfNeeded();
1200
- }
1201
- }
1202
- // Cancel any dust created by post-reset fills immediately.
1203
- if (!this._shuttingDown) {
1204
- try {
1205
- const postResetHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
1206
- await this._cancelDustOrders({
1207
- buy: postResetHealth.buyDustOrders,
1208
- sell: postResetHealth.sellDustOrders,
1209
- });
1210
- }
1211
- catch (_dustErr) {
1212
- this._warn(`[POST-RESET] Dust cancel failed: ${_dustErr.message}`);
1213
- }
1214
- }
1215
- this._log('Bootstrap phase complete - fill processing resumed', 'info');
1216
- });
1217
- await this._setupTriggerFileDetection();
1218
- await this._setupCreditRuntime();
1219
- await this._refreshAndSyncCreditRuntime();
1220
- this._setupBlockchainFetchInterval();
1221
- this._setupCreditWatchdogInterval();
1222
- this._setupCredentialDaemonWatchdogInterval();
1223
- this._setupDustHealthCheckInterval();
1224
- await this._runDustHealthCheck();
1225
- this._log('[DUST] Startup health check complete');
1226
- if (this._isOpenOrdersSyncLoopEnabled()) {
1227
- this._startOpenOrdersSyncLoop();
1228
- }
1229
- else {
1230
- this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
1231
- }
1232
- this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
1233
- return; // Skip normal startup path
1234
- }
1235
- // Restore persisted BTS fee
1236
- // SAFE: Done at startup before orders are created, and within fill lock when needed
1237
- this.manager.resetFunds();
1238
- // CRITICAL FIX: Restore BTS fees owed from persistence
1239
- if (persistedBtsFeesOwed && persistedBtsFeesOwed > 0) {
1240
- this.manager.funds.btsFeesOwed = Number(persistedBtsFeesOwed);
1241
- }
1242
- // Restore BTS balance for non-BTS pairs
1243
- if (this.config.assetA !== 'BTS' && this.config.assetB !== 'BTS') {
1244
- if (persistedBtsBalance && typeof persistedBtsBalance === 'object') {
1245
- this.manager.btsBalance = {
1246
- free: persistedBtsBalance.free || 0,
1247
- total: persistedBtsBalance.total || 0,
1248
- locked: persistedBtsBalance.locked || 0
1249
- };
1250
- }
1251
- }
1252
- // Restore recently queued fill keys for crash-durable dedup window.
1253
- // All persisted keys are restored; the in-memory TTL (_fillDedupeWindowMs)
1254
- // will naturally evict any that are already expired on the next fill cycle.
1255
- if (persistedRecentFillKeys && typeof persistedRecentFillKeys === 'object') {
1256
- for (const [fillKey, timestamp] of Object.entries(persistedRecentFillKeys)) {
1257
- this._recentlyQueuedFills.set(fillKey, Number(timestamp));
1258
- }
1259
- this._log(`Restored ${Object.keys(persistedRecentFillKeys).length} recently queued fill key(s) from persisted snapshot`, 'debug');
1260
- }
1261
- if (!this.config.dryRun && !this.accountId) {
1262
- throw new Error('Cannot start bot without a resolved account ID');
1263
- }
1264
- // Use this.accountId which was set during initialize()
1265
- const chainOpenOrders = this.config.dryRun ? [] : await chainOrders.readOpenOrders(this.accountId);
1266
- let shouldRegenerate = false;
1267
- if (!persistedGrid || persistedGrid.length === 0) {
1268
- shouldRegenerate = true;
1269
- this._log('No persisted grid found. Generating new grid.');
1270
- }
1271
- else {
1272
- await this.manager._initializeAssets();
1273
- const decision = await decideStartupGridAction({
1274
- persistedGrid,
1275
- chainOpenOrders,
1276
- manager: this.manager,
1277
- logger: { log: (msg) => this._log(msg) },
1278
- storeGrid: async (orders) => {
1279
- // Pass the snapshot directly to persistGrid so the live
1280
- // `manager.orders` map is not briefly swapped (which would
1281
- // leave the _ordersByState/_ordersByType indexes inconsistent
1282
- // with the map for the duration of the persist call).
1283
- await this.manager.persistGrid(orders);
1284
- },
1285
- attemptResumeFn: attemptResumePersistedGridByPriceMatch,
1286
- });
1287
- shouldRegenerate = decision.shouldRegenerate;
1288
- if (shouldRegenerate && chainOpenOrders.length === 0) {
1289
- this._log('Persisted grid found, but no matching active orders on-chain. Generating new grid.');
1290
- }
1291
- // Also log when regeneration is needed but other pairs' orders
1292
- // suppress the "no matching orders" message above.
1293
- if (shouldRegenerate && chainOpenOrders.length > 0 && this.manager?.assets) {
1294
- const orderCount = chainOpenOrders.filter(o => parseChainOrder(o, this.manager.assets) !== null).length;
1295
- if (orderCount === 0) {
1296
- this._log(`Persisted grid found with no matching orders (${chainOpenOrders.length} other-pair order(s) on account). Generating new grid.`);
1297
- }
1298
- }
1299
- }
1300
- // Restore BTS fees owed ONLY if we're NOT regenerating the grid
1301
- if (!shouldRegenerate) {
1302
- // CRITICAL: Restore BTS fees owed from blockchain operations
1303
- if (persistedBtsFeesOwed > 0) {
1304
- this.manager.funds.btsFeesOwed = persistedBtsFeesOwed;
1305
- this._log(`✓ Restored BTS fees owed: ${Format.formatAmount8(persistedBtsFeesOwed)} BTS`);
1306
- }
1307
- }
1308
- else {
1309
- this._log(`ℹ Grid regenerating - resetting BTS fees to clean state`);
1310
- this.manager.funds.btsFeesOwed = 0;
1311
- }
1312
- // CRITICAL: Use fill lock during ENTIRE startup synchronization to prevent races.
1313
- // This includes grid init, finishBootstrap, and maintenance - all in one atomic block.
1314
- // Lock order: _fillProcessingLock → _divergenceLock (canonical order, same as _consumeFillQueue)
1315
- await this.manager._fillProcessingLock.acquire(async () => {
1316
- try {
1317
- this._refreshDynamicWeightDistribution('startup');
1318
- if (shouldRegenerate) {
1319
- await this.manager._initializeAssets();
1320
- if (Array.isArray(chainOpenOrders) && chainOpenOrders.length > 0) {
1321
- this._log('Generating new grid and syncing with existing on-chain orders...');
1322
- await Grid.initializeGrid(this.manager);
1323
- await this.manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true });
1324
- const rebalanceResult = await reconcileGridOrders({
1325
- manager: this.manager,
1326
- config: this.config,
1327
- account: this.account,
1328
- privateKey: this.privateKey,
1329
- chainOrders,
1330
- chainOpenOrders,
1331
- });
1332
- await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (regenerated grid)');
1333
- }
1334
- else {
1335
- this._log('Generating new grid and placing initial orders on-chain...');
1336
- await this.placeInitialOrders();
1337
- }
1338
- await this._persistAndRecoverIfNeeded();
1339
- }
1340
- else {
1341
- this._log('Found active session. Loading and syncing existing grid.');
1342
- await Grid.loadGrid(this.manager, persistedGrid, persistedBoundaryIdx);
1343
- let startupChainOpenOrders = chainOpenOrders;
1344
- const syncResult = await this.manager.syncFromOpenOrders(startupChainOpenOrders, { skipAccounting: true });
1345
- // Process price corrections queued during startup sync.
1346
- // These are not picked up by _consumeFillQueue until a fill
1347
- // arrives, which may never come for an idle market.
1348
- if (syncResult.ordersNeedingCorrection?.length > 0) {
1349
- await correctAllPriceMismatches(this.manager, this.account, this.privateKey, chainOrders);
1350
- }
1351
- if (syncResult.filledOrders && syncResult.filledOrders.length > 0) {
1352
- this._log(`Startup sync: ${syncResult.filledOrders.length} grid order(s) found filled. Processing proceeds.`, 'info');
1353
- const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), 'startup sync fill rebalance', { skipAccountTotalsUpdate: true });
1354
- if (!batchResult?.aborted) {
1355
- // Refresh open orders so startup reconcile works with post-batch chain reality
1356
- // and avoids reconciling against a stale pre-batch snapshot.
1357
- startupChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
1358
- await this.manager.synchronizeWithChain(startupChainOpenOrders, 'readOpenOrders');
1359
- }
1360
- }
1361
- const rebalanceResult = await reconcileGridOrders({
1362
- manager: this.manager,
1363
- config: this.config,
1364
- account: this.account,
1365
- privateKey: this.privateKey,
1366
- chainOrders,
1367
- chainOpenOrders: startupChainOpenOrders,
1368
- });
1369
- await this._executeBatchIfNeeded(rebalanceResult, 'startup reconcile (loaded grid)');
1370
- // Dust state is no longer persisted — cancelled immediately on detection.
1371
- await this._persistAndRecoverIfNeeded();
1372
- // Gap 1: Grid snapshot sanity check — shared logic with recovery path.
1373
- await this._rejectCorruptedGridSnapshot('startup');
1374
- }
1375
- // Drain any fills that arrived during startup while still in bootstrap
1376
- // mode. Safe to call directly since we already hold _fillProcessingLock
1377
- // and _processFillsWithBootstrapMode does NOT re-acquire it.
1378
- if (this._incomingFillQueue.length > 0) {
1379
- this._log(`[STARTUP] Processing ${this._incomingFillQueue.length} queued fill(s) before bootstrap ends`);
1380
- await this._processFillsWithBootstrapMode(chainOrders);
1381
- }
1382
- this.manager.finishBootstrap();
1383
- // Refresh account totals after bootstrap to eliminate the timing
1384
- // gap between the initial balance fetch and grid operations (sync,
1385
- // reconcile, fills). Without this, the first maintenance cycle sees
1386
- // a fund drift (expected at bootstrap) and triggers an unnecessary
1387
- // invariant violation + full recovery cycle.
1388
- // Bound by a timeout to avoid blocking _fillProcessingLock on a
1389
- // flaky node. If the fetch times out, continue with cached values;
1390
- // the next periodic maintenance cycle will retry.
1391
- const FETCH_TIMEOUT_MS = 30000;
1392
- let _fetchTimeoutHandle;
1393
- try {
1394
- await Promise.race([
1395
- this.manager.fetchAccountTotals(),
1396
- new Promise((_, reject) => {
1397
- _fetchTimeoutHandle = setTimeout(() => reject(new Error('timeout')), FETCH_TIMEOUT_MS);
1398
- })
1399
- ]);
1400
- }
1401
- catch (fetchErr) {
1402
- this._log(`[STARTUP] [${this.config?.botKey || 'unknown'}] fetchAccountTotals ${fetchErr.message === 'timeout' ? 'timed out' : 'failed'} (${fetchErr.message}). Continuing with cached account totals.`, 'warn');
1403
- }
1404
- finally {
1405
- clearTimeout(_fetchTimeoutHandle);
1406
- }
1407
- // Perform initial grid maintenance (thresholds, divergence, spread, health)
1408
- // Consolidated into shared logic to ensure consistent behavior at boot and runtime.
1409
- // CRITICAL: Pass lockAlreadyHeld since we're inside _fillProcessingLock.acquire()
1410
- await this._runGridMaintenance('startup');
1411
- // Cancel any dust from a previous bot lifetime immediately.
1412
- const startupHealth = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
1413
- await this._cancelDustOrders({
1414
- buy: startupHealth.buyDustOrders,
1415
- sell: startupHealth.sellDustOrders,
1416
- });
1417
- this._log('Bootstrap phase complete - fill processing resumed', 'info');
1418
- }
1419
- finally {
1420
- // CRITICAL: Always clear bootstrap flag, even on error
1421
- this.manager.finishBootstrap();
1422
- }
1423
- });
1424
- await this._setupTriggerFileDetection();
1425
- await this._setupCreditRuntime();
1426
- await this._refreshAndSyncCreditRuntime();
1427
- await this._runCreditRuntimeMaintenance('startup');
1428
- this._setupBlockchainFetchInterval();
1429
- this._setupCreditWatchdogInterval();
1430
- this._setupCredentialDaemonWatchdogInterval();
1431
- // Periodic dust health check — catches partials that fell below threshold
1432
- // without triggering the post-fill pipeline. Cancels immediately inside the
1433
- // fill-processing lock to avoid racing with fill batches or sync operations.
1434
- this._setupDustHealthCheckInterval();
1435
- await this._runDustHealthCheck();
1436
- this._log('[DUST] Startup health check complete');
1437
- if (this._isOpenOrdersSyncLoopEnabled()) {
1438
- this._startOpenOrdersSyncLoop();
1439
- }
1440
- else {
1441
- this._log('Open-orders sync loop disabled by configuration (TIMING.OPEN_ORDERS_SYNC_LOOP_ENABLED=false)');
1442
- }
1443
- this._log(`DEXBot started. OrderManager running (dryRun=${!!this.config.dryRun})`);
1444
- }
1445
- catch (err) {
1446
- this._warn(`Error during grid initialization: ${err.message}`);
1447
- await this.shutdown();
1448
- throw err;
1449
- }
595
+ return DexbotStartupRuntime.finishStartupSequence(this, startupState);
1450
596
  }
1451
597
  /**
1452
598
  * Create the fill callback for listenForFills.
@@ -1465,35 +611,10 @@ class DEXBot {
1465
611
  * @returns {Promise<{syncResult: Object|null, aborted: boolean, hasUnmatched: number, openOrders: Array|null}>}
1466
612
  */
1467
613
  async _syncOpenOrdersAndProcessFills(tag) {
1468
- if (!this.accountId || this.config?.dryRun) {
1469
- return { syncResult: null, aborted: false, hasUnmatched: 0, openOrders: null };
1470
- }
1471
- try {
1472
- let openOrders = await chainOrders.readOpenOrders(this.accountId);
1473
- const syncResult = await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
1474
- let aborted = false;
1475
- if (syncResult?.filledOrders?.length > 0) {
1476
- this._refreshDynamicWeightDistribution(`${tag} sync-fill`);
1477
- this._log(`[SYNC-CHAIN] ${syncResult.filledOrders.length} filled order(s) found during ${tag}`, 'info');
1478
- const batchResult = await this._processFillsWithBatching(syncResult.filledOrders, new Set(), `${tag} sync-fill`);
1479
- if (!batchResult?.aborted) {
1480
- openOrders = await chainOrders.readOpenOrders(this.accountId);
1481
- await this.manager.synchronizeWithChain(openOrders, 'readOpenOrders');
1482
- }
1483
- else {
1484
- aborted = true;
1485
- }
1486
- }
1487
- const hasUnmatched = syncResult?.unmatchedChainOrders?.length || 0;
1488
- return { syncResult, aborted, hasUnmatched, openOrders };
1489
- }
1490
- catch (err) {
1491
- this._warn(`[SYNC-CHAIN] Open-orders sync failed during ${tag}: ${err.message}`);
1492
- return { syncResult: null, aborted: true, hasUnmatched: -1, openOrders: null };
1493
- }
614
+ return DexbotMaintenanceRuntime.syncOpenOrdersAndProcessFills(this, tag);
1494
615
  }
1495
616
  _maxConsecutiveFillConsumerFailures() {
1496
- return this.config.fillProcessing?.MAX_CONSECUTIVE_CONSUMER_FAILURES ?? FILL_PROCESSING.MAX_CONSECUTIVE_CONSUMER_FAILURES;
617
+ return DexbotFillRuntime.maxConsecutiveFillConsumerFailures(this);
1497
618
  }
1498
619
  /**
1499
620
  * Compute the backoff delay for fill-consumer retries after the failure
@@ -1505,80 +626,10 @@ class DEXBot {
1505
626
  * @private
1506
627
  */
1507
628
  _computeFillConsumerBackoffMs(failures) {
1508
- const fp = this.config.fillProcessing || FILL_PROCESSING;
1509
- const initial = fp.CONSUMER_BACKOFF_INITIAL_MS;
1510
- const max = fp.CONSUMER_BACKOFF_MAX_MS;
1511
- const stepAfterMax = Math.max(0, failures - this._maxConsecutiveFillConsumerFailures());
1512
- // 0 -> initial, 1 -> 2*initial, 2 -> 4*initial, ... capped at max.
1513
- return Math.min(max, initial * Math.pow(2, stepAfterMax));
629
+ return DexbotFillRuntime.computeFillConsumerBackoffMs(this, failures);
1514
630
  }
1515
631
  _scheduleFillConsumerRestart(chainOrders) {
1516
- const failures = this._consecutiveConsumeFailures;
1517
- if (failures >= this._maxConsecutiveFillConsumerFailures()) {
1518
- // Past the failure budget: switch from tight setImmediate loop to
1519
- // exponential backoff. The consumer continues to retry — a transient
1520
- // outage (e.g., credential daemon recovery) will resume normal
1521
- // operation as soon as one cycle succeeds and resets the counter.
1522
- // A permanent failure mode yields slower-but-still-progressing
1523
- // retries capped at CONSUMER_BACKOFF_MAX_MS, with no permanent
1524
- // stop that would require a bot restart.
1525
- const backoffMs = this._computeFillConsumerBackoffMs(failures);
1526
- const elapsedSec = this._consumeFailureFirstAt
1527
- ? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
1528
- : null;
1529
- const elapsed = elapsedSec !== null ? `${elapsedSec}s` : 'unknown';
1530
- // Escalate log level on sustained failure so operators monitoring
1531
- // for error/critical alerts are not blind to a stuck consumer.
1532
- // - warn : within the first escalation window (5-9 failures,
1533
- // <5 min) — could be a slow recovery
1534
- // - error : 10+ failures OR 5+ minutes of sustained failure
1535
- // - critical: 20+ failures OR 15+ minutes of sustained failure —
1536
- // this is the "permanent fault" signal
1537
- const sustainedLevel = (failures >= 20 || (elapsedSec !== null && elapsedSec >= 900))
1538
- ? 'critical'
1539
- : (failures >= 10 || (elapsedSec !== null && elapsedSec >= 300))
1540
- ? 'error'
1541
- : 'warn';
1542
- this._log(`[FILL-QUEUE] Fill consumer has failed ${failures} consecutive times over ${elapsed}; ` +
1543
- `backing off ${Math.round(backoffMs / TIMING.MILLISECONDS_PER_SECOND)}s before retry. ` +
1544
- `Queue: ${this._incomingFillQueue.length} fills.`, sustainedLevel);
1545
- setTimeout(() => {
1546
- if (this._shuttingDown)
1547
- return;
1548
- this._consumeFillQueue(chainOrders).catch(err => {
1549
- if (!this._consumeFailureFirstAt) {
1550
- this._consumeFailureFirstAt = Date.now();
1551
- }
1552
- this._consecutiveConsumeFailures++;
1553
- const newFailures = this._consecutiveConsumeFailures;
1554
- const newElapsedSec = this._consumeFailureFirstAt
1555
- ? Math.round((Date.now() - this._consumeFailureFirstAt) / TIMING.MILLISECONDS_PER_SECOND)
1556
- : null;
1557
- const resumeLevel = (newFailures >= 20 || (newElapsedSec !== null && newElapsedSec >= 900))
1558
- ? 'critical'
1559
- : (newFailures >= 10 || (newElapsedSec !== null && newElapsedSec >= 300))
1560
- ? 'error'
1561
- : 'warn';
1562
- this._log(`Fill consumer resume after backoff failed ` +
1563
- `(${newFailures} total, ` +
1564
- `next backoff ${Math.round(this._computeFillConsumerBackoffMs(newFailures) / TIMING.MILLISECONDS_PER_SECOND)}s): ` +
1565
- `${err.message}`, resumeLevel);
1566
- // Continue the backoff loop. The success path of
1567
- // _consumeFillQueue resets the counter, breaking the cycle.
1568
- this._scheduleFillConsumerRestart(chainOrders);
1569
- });
1570
- }, backoffMs);
1571
- return;
1572
- }
1573
- setImmediate(() => this._consumeFillQueue(chainOrders).catch(err => {
1574
- if (!this._consumeFailureFirstAt) {
1575
- this._consumeFailureFirstAt = Date.now();
1576
- }
1577
- this._consecutiveConsumeFailures++;
1578
- const remaining = this._maxConsecutiveFillConsumerFailures() - this._consecutiveConsumeFailures;
1579
- this._log(`Fill consumer failed (${this._consecutiveConsumeFailures}/${this._maxConsecutiveFillConsumerFailures()}, ` +
1580
- `${remaining} attempts remaining): ${err.message}`, this._consecutiveConsumeFailures >= 3 ? 'warn' : 'error');
1581
- }));
632
+ DexbotFillRuntime.scheduleFillConsumerRestart(this, chainOrders);
1582
633
  }
1583
634
  /**
1584
635
  * Consume queued fills from incomingFillQueue and rebalance.
@@ -1594,564 +645,7 @@ class DEXBot {
1594
645
  * @private
1595
646
  */
1596
647
  async _consumeFillQueue(chainOrders) {
1597
- // Helper: every early return below is a "deferral", not a failure.
1598
- // The counter only tracks actual failures, so any healthy deferral
1599
- // path should also reset the counter. Without this, a sequence of
1600
- // F-S-F-S-F-S-F-S-F (fail, succeed, fail, succeed, ...) would still
1601
- // reach the max and trip backoff, OR a failure followed by an empty
1602
- // queue / shutdown / in-flight batch would leave the counter sticky
1603
- // and trigger backoff one step sooner on the next real failure.
1604
- const resetFailureWatchdogIfSet = () => {
1605
- if (this._consecutiveConsumeFailures > 0 || this._consumeFailureFirstAt > 0) {
1606
- this._consecutiveConsumeFailures = 0;
1607
- this._consumeFailureFirstAt = 0;
1608
- }
1609
- };
1610
- // ATOMIC: Only attempt lock acquisition if queue has work
1611
- // This prevents unnecessary lock contention on empty queues
1612
- if (this._incomingFillQueue.length === 0) {
1613
- // Empty queue = consumer is healthy, just idle.
1614
- resetFailureWatchdogIfSet();
1615
- return;
1616
- }
1617
- // Check shutdown state
1618
- if (this._shuttingDown) {
1619
- this._warn('Fill processing skipped: shutdown in progress');
1620
- resetFailureWatchdogIfSet();
1621
- return;
1622
- }
1623
- if (this._batchInFlight || this._recoverySyncInFlight) {
1624
- this.manager?.logger?.log?.(`Fill processing deferred: order pipeline active (${this._incomingFillQueue.length} queued)`, 'debug');
1625
- // A batch is in flight, not a failure. The next iteration will
1626
- // either succeed (and reset the counter on the success path) or
1627
- // fail and increment it. Either way, leaving an old failure
1628
- // count here would double-count.
1629
- resetFailureWatchdogIfSet();
1630
- return;
1631
- }
1632
- let pendingFillKeysForCurrentCycle = new Set();
1633
- try {
1634
- // BOOTSTRAP OPTIMIZATION: During bootstrap, prioritize fill processing over grid-wide checks
1635
- // Process fills immediately with side-only rebalancing (no expensive full grid recalculations)
1636
- if (this.manager.isBootstrapping()) {
1637
- // During bootstrap: skip lock contention checks, process fills directly
1638
- let bootstrapSkipped = false;
1639
- await this.manager._fillProcessingLock.acquire(async () => {
1640
- if (!this.manager.isBootstrapping()) {
1641
- // Bootstrap finished while waiting for the lock — no
1642
- // work to do, but the iteration is still healthy.
1643
- bootstrapSkipped = true;
1644
- return;
1645
- }
1646
- await this._processFillsWithBootstrapMode(chainOrders);
1647
- });
1648
- if (bootstrapSkipped) {
1649
- // Bootstrap-mode lock callback returned without doing
1650
- // work; the .acquire() success path at line ~1941 that
1651
- // would normally reset the counter is not reached in
1652
- // this branch. Reset here so a stale counter from a
1653
- // prior failure doesn't carry over.
1654
- resetFailureWatchdogIfSet();
1655
- }
1656
- return;
1657
- }
1658
- // NORMAL MODE: Non-blocking check if lock already has waiters
1659
- // This prevents unbounded queue growth while still ensuring processing
1660
- // Note: We DO proceed if lock is held but has no waiters - we'll wait our turn
1661
- if (this.manager._fillProcessingLock.getQueueLength() > 0) {
1662
- this._metrics.lockContentionEvents++;
1663
- // Deferral, not a failure. Reset the watchdog so the next
1664
- // call (which may now find an empty queue, or process
1665
- // successfully) doesn't inherit a stale counter.
1666
- resetFailureWatchdogIfSet();
1667
- return;
1668
- }
1669
- await this.manager._fillProcessingLock.acquire(async () => {
1670
- // Reset orphan-fill credit timestamp at the start of each
1671
- // fill cycle. It is re-set when orphan fills are credited,
1672
- // and consumed (set to null) on the next fund-invariant check
1673
- // in accounting.ts, which widens tolerance by 5x while set.
1674
- // Written to this.manager (not this) because accounting.ts
1675
- // reads from the OrderManager reference (mgr).
1676
- // Also cleared by _performStateRecovery after a fresh chain
1677
- // fetch. The timestamp value itself is not compared against a
1678
- // window — it acts as a consume-on-read boolean.
1679
- this.manager._orphanFillsCreditedAt = null;
1680
- while (this._incomingFillQueue.length > 0) {
1681
- const batchStartTime = Date.now();
1682
- // Track max queue depth
1683
- this._metrics.maxQueueDepth = Math.max(this._metrics.maxQueueDepth, this._incomingFillQueue.length);
1684
- // 1. Take snapshot of current work (ATOMIC: splice removes and returns fills atomically)
1685
- const allFills = this._incomingFillQueue.splice(0); // Atomically clear and get all fills
1686
- const validFills = [];
1687
- const processedFillKeys = new Set();
1688
- pendingFillKeysForCurrentCycle = new Set();
1689
- let requiresOpenOrdersSync = false;
1690
- // 2. Filter and Deduplicate (Standard Logic)
1691
- for (const fill of allFills) {
1692
- if (fill && fill.op && fill.op[0] === FILL_PROCESSING.OPERATION_TYPE) {
1693
- const fillOp = fill.op[1];
1694
- // SELF-CANCEL GUARD: only drop malformed, cancel-like
1695
- // artifacts for an order the local process just cancelled.
1696
- // Real fill_order ops carry economic data and must still be
1697
- // accounted even if they arrive shortly after a successful
1698
- // cancel broadcast.
1699
- const hasFillEconomics = fillOp?.pays?.asset_id && fillOp?.pays?.amount != null
1700
- && fillOp?.receives?.asset_id && fillOp?.receives?.amount != null;
1701
- if (chainOrders && typeof chainOrders.wasRecentlyOwnCancelled === 'function'
1702
- && chainOrders.wasRecentlyOwnCancelled(fillOp.order_id)
1703
- && !hasFillEconomics) {
1704
- this.manager.logger.log(`[SELF-CANCEL] Skipping non-economic fill artifact for order ${fillOp.order_id} (just cancelled by this bot)`, 'debug');
1705
- continue;
1706
- }
1707
- // ACCOUNT VALIDATION: Verify the filled order belongs to this bot's account/grid
1708
- // Only process fills for orders we actually manage
1709
- const gridOrder = this.manager.orders.get(fillOp.order_id) ||
1710
- Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
1711
- if (!gridOrder) {
1712
- // Check if this order was already freed by stale-order batch cleanup.
1713
- // When a batch fails due to a stale order reference, the cleanup converts the
1714
- // slot to VIRTUAL/SPREAD, releasing committed funds to chainFree. If we also
1715
- // credit the fill proceeds here, we double-count the capital.
1716
- const staleMarkedAt = this._staleCleanedOrderIds.get(fillOp.order_id);
1717
- if (staleMarkedAt != null) {
1718
- const staleAgeMs = Date.now() - staleMarkedAt;
1719
- if (staleAgeMs <= this._staleCleanupRetentionMs) {
1720
- this.manager.logger.log(`[ORPHAN-FILL] Skipping double-credit for stale-cleaned order ${fillOp.order_id} ` +
1721
- `(funds already freed by batch cleanup, age=${staleAgeMs}ms)`, 'warn');
1722
- continue;
1723
- }
1724
- this._staleCleanedOrderIds.delete(fillOp.order_id);
1725
- }
1726
- // Legitimate orphan fill: order was virtualized during sequential processing
1727
- // but a fill arrived afterward. Credit proceeds to maintain fund tracking.
1728
- let orphanFillKey = buildFillKey(fill);
1729
- if (!orphanFillKey) {
1730
- orphanFillKey = this._buildOrphanFillFallbackKey(fill);
1731
- }
1732
- if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[ORPHAN-FILL]', fillOp.order_id)) {
1733
- continue;
1734
- }
1735
- this.manager.logger.log(`[ORPHAN-FILL] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
1736
- const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
1737
- context: 'ORPHAN-FILL',
1738
- replayMessage: (op) => `[ORPHAN-FILL] Replay detected for ${op.order_id}; skipping duplicate credit`
1739
- });
1740
- if (accountingResult.status === 'missing_key') {
1741
- requiresOpenOrdersSync = true;
1742
- }
1743
- // Record orphan fill credit timestamp for fund invariant
1744
- // tolerance widening. Written to this.manager because
1745
- // accounting.ts reads from the OrderManager (mgr).
1746
- this.manager._orphanFillsCreditedAt = Date.now();
1747
- // Don't add to validFills - we can't do rebalancing without a grid slot
1748
- // But the funds are now credited, preventing fund invariant violation
1749
- continue;
1750
- }
1751
- // Process both maker and taker fills for our grid orders
1752
- // Grid validation ensures we only process fills belonging to our account
1753
- // Taker fills are included because the bot may execute market orders or act as taker
1754
- const roleStr = fillOp.is_maker !== false ? 'maker' : 'taker';
1755
- this.manager.logger.log(`Processing ${roleStr} fill for order ${fillOp.order_id}`, 'debug');
1756
- const fillKey = buildFillKey(fill);
1757
- if (!fillKey) {
1758
- this.manager.logger.log(`[FILL] Missing history id for order ${fillOp.order_id} block ${fill.block_num}; deferring to open-orders sync`, 'warn');
1759
- requiresOpenOrdersSync = true;
1760
- continue;
1761
- }
1762
- if (!this._isNewFillKey(fillKey, processedFillKeys, '[FILL]', fillOp.order_id)) {
1763
- continue;
1764
- }
1765
- validFills.push(fill);
1766
- // Log info
1767
- const paysAmount = fillOp.pays ? fillOp.pays.amount : '?';
1768
- const receivesAmount = fillOp.receives ? fillOp.receives.amount : '?';
1769
- this._log(`\n===== FILL DETECTED =====`);
1770
- this._log(`Order ID: ${fillOp.order_id}`);
1771
- this._log(`Pays: ${paysAmount}, Receives: ${receivesAmount}`);
1772
- this._log(`Block: ${fill.block_num} (History ID: ${fill.id || 'N/A'})`);
1773
- this._log(`=========================\n`);
1774
- }
1775
- }
1776
- // Clean up short-lived queue dedupe cache to prevent memory leak.
1777
- const cleanupTimestamp = Date.now();
1778
- let cleanedCount = 0;
1779
- for (const [key, timestamp] of this._recentlyQueuedFills) {
1780
- if (cleanupTimestamp - timestamp > this._fillDedupeWindowMs) {
1781
- this._recentlyQueuedFills.delete(key);
1782
- cleanedCount++;
1783
- }
1784
- }
1785
- if (cleanedCount > 0) {
1786
- this.manager.logger.log(`Cleaned ${cleanedCount} old queued fill records. Remaining: ${this._recentlyQueuedFills.size}`, 'debug');
1787
- }
1788
- if (validFills.length === 0 && !requiresOpenOrdersSync)
1789
- continue; // Loop back for more
1790
- // 3. Sync and Collect Filled Orders
1791
- let allFilledOrders = [];
1792
- let ordersNeedingCorrection = [];
1793
- const pendingGhostOrders = new Set();
1794
- const fillMode = chainOrders.getFillProcessingMode();
1795
- const processValidFills = async (fillsToSync) => {
1796
- let resolvedOrders = [];
1797
- if (fillMode === 'history') {
1798
- this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (history mode)`, 'info');
1799
- // Batch mode for 2+ fills: acquires _gridLock once, batches drift refetch
1800
- if (fillsToSync.length >= 2) {
1801
- const batchResult = await this.manager.syncFromFillHistoryBatch(fillsToSync, {
1802
- persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
1803
- });
1804
- for (const fill of fillsToSync) {
1805
- const fillKey = buildFillKey({
1806
- orderId: fill?.op?.[1]?.order_id,
1807
- blockNum: fill?.block_num,
1808
- historyId: fill?.id
1809
- });
1810
- if (fillKey)
1811
- pendingFillKeysForCurrentCycle.add(fillKey);
1812
- }
1813
- if (batchResult.filledOrders)
1814
- resolvedOrders.push(...batchResult.filledOrders);
1815
- if (batchResult.requiresOpenOrdersSync)
1816
- requiresOpenOrdersSync = true;
1817
- if (batchResult.ghostOrderIds?.length > 0) {
1818
- for (const id of batchResult.ghostOrderIds) {
1819
- pendingGhostOrders.add(id);
1820
- }
1821
- }
1822
- }
1823
- else {
1824
- // Single fill: use individual per-fill path
1825
- for (const fill of fillsToSync) {
1826
- const resultHistory = await this.manager.syncFromFillHistory(fill, {
1827
- persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
1828
- });
1829
- const fillKey = buildFillKey({
1830
- orderId: fill?.op?.[1]?.order_id,
1831
- blockNum: fill?.block_num,
1832
- historyId: fill?.id
1833
- });
1834
- if (fillKey)
1835
- pendingFillKeysForCurrentCycle.add(fillKey);
1836
- // Dust is handled by post-fill detection below.
1837
- if (resultHistory.filledOrders)
1838
- resolvedOrders.push(...resultHistory.filledOrders);
1839
- if (resultHistory.requiresOpenOrdersSync)
1840
- requiresOpenOrdersSync = true;
1841
- if (resultHistory.ghostOrderId)
1842
- pendingGhostOrders.add(resultHistory.ghostOrderId);
1843
- }
1844
- }
1845
- }
1846
- if (fillMode !== 'history' || requiresOpenOrdersSync) {
1847
- if (fillMode === 'history' && requiresOpenOrdersSync) {
1848
- this.manager.logger.log('Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
1849
- }
1850
- this.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (open orders mode)`, 'info');
1851
- const chainOpenOrders = await chainOrders.readOpenOrders(this.account);
1852
- const resultOpenOrders = await this.manager.syncFromOpenOrders(chainOpenOrders);
1853
- // Dust is handled by post-fill detection below.
1854
- if (resultOpenOrders.filledOrders)
1855
- resolvedOrders.push(...resultOpenOrders.filledOrders);
1856
- if (resultOpenOrders.ordersNeedingCorrection)
1857
- ordersNeedingCorrection.push(...resultOpenOrders.ordersNeedingCorrection);
1858
- }
1859
- return resolvedOrders;
1860
- };
1861
- this.manager.pauseFundRecalc();
1862
- try {
1863
- // FIX 1: Block-level fill batching — group valid fills by block
1864
- // and process each block group as a unit. This prevents slot
1865
- // collisions when fills from the same block (original + replacement
1866
- // orders both filled) arrive in different sync batches. Processing
1867
- // all fills from a block together lets the sync engine see every
1868
- // fill for overlapping slots simultaneously.
1869
- const fillsByBlock = new Map();
1870
- const fillsWithoutBlock = [];
1871
- for (const fill of validFills) {
1872
- if (fill.block_num != null) {
1873
- const list = fillsByBlock.get(fill.block_num);
1874
- if (list)
1875
- list.push(fill);
1876
- else
1877
- fillsByBlock.set(fill.block_num, [fill]);
1878
- }
1879
- else {
1880
- fillsWithoutBlock.push(fill);
1881
- }
1882
- }
1883
- // Process block groups in ascending block order so the sync
1884
- // engine sees a deterministic, chronological fill sequence.
1885
- const sortedBlocks = [...fillsByBlock.keys()].sort((a, b) => a - b);
1886
- const accumulatedOrders = [];
1887
- let anyRequiresSync = false;
1888
- // Capture whether any fill was skipped during validation
1889
- // (e.g. missing history ID) and set the sync flag. The
1890
- // per-block reset below would lose this state since those
1891
- // fills were filtered out of validFills and never enter
1892
- // any block group.
1893
- const initialRequiresSync = requiresOpenOrdersSync;
1894
- for (const blockNum of sortedBlocks) {
1895
- // Reset requiresOpenOrdersSync per block group so one
1896
- // block's history-id gap doesn't force the next block
1897
- // into an unnecessary open-orders snapshot re-fetch.
1898
- requiresOpenOrdersSync = false;
1899
- this.manager.logger.log(`[FILL-BLOCK] Processing ${fillsByBlock.get(blockNum).length} fill(s) from block ${blockNum}`, 'debug');
1900
- const blockResult = await processValidFills(fillsByBlock.get(blockNum));
1901
- accumulatedOrders.push(...blockResult);
1902
- if (requiresOpenOrdersSync)
1903
- anyRequiresSync = true;
1904
- }
1905
- // Preserve both per-block sync flags and the initial flag
1906
- // from filtered-out fills (missing history ID).
1907
- requiresOpenOrdersSync = anyRequiresSync || initialRequiresSync;
1908
- if (fillsWithoutBlock.length > 0) {
1909
- this.manager.logger.log(`[FILL-BLOCK] Processing ${fillsWithoutBlock.length} fill(s) without block info`, 'debug');
1910
- const noBlockResult = await processValidFills(fillsWithoutBlock);
1911
- accumulatedOrders.push(...noBlockResult);
1912
- if (requiresOpenOrdersSync)
1913
- anyRequiresSync = true;
1914
- }
1915
- // If a fill was filtered out during validation (e.g. missing
1916
- // history ID) and no block group triggered the open-orders
1917
- // sync, run it now so the grid state is reconciled.
1918
- if (requiresOpenOrdersSync && !anyRequiresSync) {
1919
- this.manager.logger.log('[FILL-BLOCK] Running open-orders sync for fills with missing history identifiers', 'warn');
1920
- const fallbackOrders = await processValidFills([]);
1921
- accumulatedOrders.push(...fallbackOrders);
1922
- // Flag consumed — variable goes out of scope on
1923
- // next while iteration.
1924
- }
1925
- allFilledOrders = accumulatedOrders;
1926
- // 4. Handle Price Corrections
1927
- if (ordersNeedingCorrection.length > 0) {
1928
- const correctionResult = await correctAllPriceMismatches(this.manager, this.account, this.privateKey, chainOrders);
1929
- if (correctionResult.failed > 0)
1930
- this.manager.logger.log(`${correctionResult.failed} corrections failed`, 'error');
1931
- }
1932
- }
1933
- finally {
1934
- // 4b. Cancel orphaned chain orders detected by other-side rounding.
1935
- // These orders have a tiny residual that the bot treated as fully
1936
- // filled, but the blockchain did not close the order. We send a
1937
- // cancel tx to clean up the zombie order on chain.
1938
- // Runs in the finally so it still fires even if processValidFills
1939
- // throws after collecting ghost IDs into pendingGhostOrders.
1940
- //
1941
- // OPTIMIZATION: Batch all new ghost cancels into one executeBatch
1942
- // call with multiple cancelOrder ops, reducing per-order tx fees
1943
- // and broadcast overhead. Falls back to individual cancels on
1944
- // batch failure.
1945
- if (pendingGhostOrders.size > 0) {
1946
- if (!this._ghostOrderCancelAttempted)
1947
- this._ghostOrderCancelAttempted = new Set();
1948
- const newGhostIds = [...pendingGhostOrders].filter(id => !this._ghostOrderCancelAttempted.has(id));
1949
- if (newGhostIds.length > 0) {
1950
- const MAX_OPS_PER_TX = 200;
1951
- let batchFailed = false;
1952
- // Build cancel ops with per-ID error tolerance.
1953
- // Index correlates 1:1 with newGhostIds (allSettled preserves order).
1954
- const buildResults = await Promise.allSettled(newGhostIds.map(id => chainOrders.buildCancelOrderOp(this.account, id)));
1955
- const cancelOps = [];
1956
- for (let i = 0; i < buildResults.length; i++) {
1957
- const result = buildResults[i];
1958
- const id = newGhostIds[i];
1959
- if (result.status === 'fulfilled') {
1960
- cancelOps.push(result.value);
1961
- }
1962
- else {
1963
- this.manager.logger.log(`[SYNC] Failed to build cancel op for ghost order ${id}: ${result.reason?.message || result.reason}`, 'warn');
1964
- }
1965
- }
1966
- // Chunk into batches respecting MAX_OPS_PER_TX
1967
- for (let i = 0; i < cancelOps.length; i += MAX_OPS_PER_TX) {
1968
- const chunk = cancelOps.slice(i, i + MAX_OPS_PER_TX);
1969
- const batchIds = newGhostIds.slice(i, i + MAX_OPS_PER_TX);
1970
- try {
1971
- this.manager.logger.log(`[SYNC] Batch-cancelling ${chunk.length} orphaned chain order(s) ` +
1972
- `(batch ${Math.floor(i / MAX_OPS_PER_TX) + 1}/${Math.ceil(cancelOps.length / MAX_OPS_PER_TX)})`, 'info');
1973
- await chainOrders.executeBatch(this.account, this.privateKey, chunk);
1974
- // Mark successfully cancelled IDs immediately
1975
- for (const id of batchIds) {
1976
- this._ghostOrderCancelAttempted.add(id);
1977
- }
1978
- }
1979
- catch (batchErr) {
1980
- batchFailed = true;
1981
- this.manager.logger.log(`[SYNC] Batch ghost cancel failed for chunk ${Math.floor(i / MAX_OPS_PER_TX) + 1} ` +
1982
- `(${chunk.length} orders): ${batchErr?.message || batchErr}`, 'warn');
1983
- }
1984
- }
1985
- // Log overall success if all chunks and all builds succeeded
1986
- if (!batchFailed && cancelOps.length === newGhostIds.length) {
1987
- this.manager.logger.log(`[SYNC] Successfully batch-cancelled ${newGhostIds.length} orphaned chain order(s)`, 'info');
1988
- }
1989
- // Fallback: individual cancels for IDs that were not marked successful
1990
- for (const ghostOrderId of newGhostIds) {
1991
- if (this._ghostOrderCancelAttempted.has(ghostOrderId))
1992
- continue;
1993
- try {
1994
- this.manager.logger.log(`[SYNC] Cancelling orphaned chain order ${ghostOrderId} (other-side full-fill residual)`, 'info');
1995
- await chainOrders.cancelOrder(this.account, this.privateKey, ghostOrderId);
1996
- this._ghostOrderCancelAttempted.add(ghostOrderId);
1997
- }
1998
- catch (err) {
1999
- this.manager.logger.log(`[SYNC] Failed to cancel orphaned order ${ghostOrderId}: ${err?.message || err}`, 'warn');
2000
- }
2001
- }
2002
- }
2003
- }
2004
- await this.manager.resumeFundRecalc();
2005
- }
2006
- // Refresh dynamic weight distribution before processing fills
2007
- // so the rebalance uses the latest market adapter weights, not
2008
- // stale values from the last periodic refresh cycle.
2009
- // Lightweight: reads local JSON snapshot file.
2010
- this._refreshDynamicWeightDistribution('fill queue');
2011
- // 5. Fixed-Cap Fill Rebalance
2012
- // - 1..MAX_FILL_BATCH_SIZE fills: unified full-set planning
2013
- // - larger bursts: fixed-size chunking at MAX_FILL_BATCH_SIZE
2014
- if (allFilledOrders.length > 0) {
2015
- const result = await this._processFillsWithBatching(allFilledOrders, null, 'fill set');
2016
- let abortedFillCycle = result.aborted;
2017
- if (!abortedFillCycle) {
2018
- const batchFillKeys = new Set(allFilledOrders.map(filledOrder => buildFillKey({
2019
- orderId: filledOrder?.orderId,
2020
- blockNum: filledOrder?.blockNum,
2021
- historyId: filledOrder?.historyId
2022
- })).filter(Boolean));
2023
- await this._flushProcessedFillPersistenceForKeys(batchFillKeys, 'fill-batch-committed');
2024
- }
2025
- else {
2026
- this.manager.logger.log('[FILL-DEDUP] Fill cycle aborted; fill key persistence guarded under abort path.', 'warn');
2027
- }
2028
- // 6. Rebalance Recovery Loop (Sequential Extensions)
2029
- // DISABLED FOR SEQUENTIAL: Each sequential fill already triggers a full rebalance with proper
2030
- // boundary shift. An additional recovery loop with EMPTY fills causes the boundary to remain
2031
- // at the last fill's position, leading to wrong operation types (updates instead of rotations)
2032
- // and operations on the wrong side.
2033
- //
2034
- // In the future, recovery loop can be re-enabled for single fills if needed, but ONLY
2035
- // if it passes the actual fills to processFilledOrders so the boundary shifts correctly.
2036
- // For now: Each fill = full rebalance with boundary shift = complete correction in one pass.
2037
- // CRITICAL: Do NOT run spread correction here during sequential fill processing.
2038
- // The rebalance from each fill should maintain spread naturally. Running spread correction
2039
- // immediately after creates new orders that may get filled by market before next cycle,
2040
- // causing cascading fills and potentially SPREAD slots becoming PARTIAL (error condition).
2041
- // Spread correction runs in the main loop instead.
2042
- const fullFillCount = allFilledOrders.filter(o => o && o.isPartial !== true).length;
2043
- const hasAnyFills = allFilledOrders.some(o => o);
2044
- const shouldRunPostFillChecks = !abortedFillCycle && fullFillCount > 0;
2045
- const shouldRunDustDetection = !abortedFillCycle && hasAnyFills;
2046
- if (shouldRunDustDetection) {
2047
- const healthResult = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan.bind(this));
2048
- const allDust = [
2049
- ...(healthResult.buyDustOrders || []),
2050
- ...(healthResult.sellDustOrders || []),
2051
- ];
2052
- if (allDust.length > 0) {
2053
- const dustCancelResult = await this._cancelDustOrders({
2054
- buy: healthResult.buyDustOrders,
2055
- sell: healthResult.sellDustOrders,
2056
- });
2057
- if (dustCancelResult?.batchResult?.aborted) {
2058
- abortedFillCycle = true;
2059
- }
2060
- }
2061
- }
2062
- // Run grid maintenance after fills to rebuild degraded grid.
2063
- // CRITICAL FIX (commit a946c33): Replaced inline divergence checks with centralized
2064
- // _runGridMaintenance call to ensure pipeline protection applies consistently.
2065
- // Before: Divergence checks ran immediately after fills, causing race-to-resize
2066
- // After: Grid maintenance waits for isPipelineEmpty() before structural changes
2067
- // Run only when the cycle contains at least one full fill.
2068
- if (shouldRunPostFillChecks && !abortedFillCycle) {
2069
- await this._runGridMaintenance('post-fill');
2070
- }
2071
- }
2072
- else if (pendingFillKeysForCurrentCycle.size > 0) {
2073
- await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, 'fill-batch-no-rotations');
2074
- }
2075
- this.manager._recentFillKeysSnapshot = this._getRecentFillKeysSnapshot();
2076
- await retryPersistenceIfNeeded(this.manager);
2077
- // Periodically clean up old fill records after processing N fills.
2078
- // Counter is protected by _fillProcessingLock during fill consumption.
2079
- this._fillCleanupCounter += validFills.length;
2080
- const cleanupThreshold = MAINTENANCE.CLEANUP_PROBABILITY > 0 && MAINTENANCE.CLEANUP_PROBABILITY < 1
2081
- ? Math.floor(1 / MAINTENANCE.CLEANUP_PROBABILITY)
2082
- : 100; // Default: every 100 fills
2083
- if (this._fillCleanupCounter >= cleanupThreshold) {
2084
- try {
2085
- await this.accountOrders.cleanOldProcessedFills(TIMING.FILL_RECORD_RETENTION_MS);
2086
- this._fillCleanupCounter = 0; // Reset counter after cleanup (success or retry on next batch if failed)
2087
- }
2088
- catch (err) {
2089
- this.manager?.logger?.log(`Warning: Fill cleanup failed (will retry): ${err.message}`, 'warn');
2090
- }
2091
- }
2092
- // Update metrics
2093
- this._metrics.fillsProcessed += validFills.length;
2094
- this._metrics.fillProcessingTimeMs += Date.now() - batchStartTime;
2095
- // Prune expired stale-cleaned order IDs after each processing cycle.
2096
- if (this._staleCleanedOrderIds.size > 0) {
2097
- const now = Date.now();
2098
- let prunedCount = 0;
2099
- for (const [orderId, markedAt] of this._staleCleanedOrderIds) {
2100
- if (now - markedAt > this._staleCleanupRetentionMs) {
2101
- this._staleCleanedOrderIds.delete(orderId);
2102
- prunedCount++;
2103
- }
2104
- }
2105
- if (prunedCount > 0) {
2106
- this.manager.logger.log(`[STALE-CLEANUP] Pruned ${prunedCount} expired stale-cleaned order IDs ` +
2107
- `(retention=${this._staleCleanupRetentionMs}ms, remaining=${this._staleCleanedOrderIds.size})`, 'debug');
2108
- }
2109
- }
2110
- } // End while(_incomingFillQueue)
2111
- this._markGridActivity('fill processing end');
2112
- // Reset the fill-consumer watchdog on the success path. The
2113
- // counter is only incremented by _scheduleFillConsumerRestart's
2114
- // catch handler; without this reset, a fill pattern of
2115
- // F-S-F-S-F-S-F-S-F would still reach the max and the consumer
2116
- // would stop being re-scheduled.
2117
- this._consecutiveConsumeFailures = 0;
2118
- this._consumeFailureFirstAt = 0;
2119
- });
2120
- }
2121
- catch (err) {
2122
- const isCredentialOutage = this._isCredentialDaemonError(err);
2123
- if (pendingFillKeysForCurrentCycle.size > 0) {
2124
- const flushReason = isCredentialOutage
2125
- ? 'credential-outage-verified-fills'
2126
- : 'fill-cycle-error-verified-fills';
2127
- if (isCredentialOutage) {
2128
- this._credentialRecoveryNeeded = true;
2129
- this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
2130
- }
2131
- try {
2132
- await this._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, flushReason, { throwOnError: true });
2133
- const credentialSuffix = isCredentialOutage
2134
- ? '; grid persistence is suspended until recovery'
2135
- : '';
2136
- this.manager?.logger?.log?.(`[FILL-DEDUP] Persisted ${pendingFillKeysForCurrentCycle.size} verified processed-fill write(s) after fill cycle error${credentialSuffix}.`, isCredentialOutage ? 'warn' : 'info');
2137
- }
2138
- catch (flushErr) {
2139
- this.manager?.logger?.log?.(`[FILL-DEDUP] Failed to persist verified fill keys during fill error handling: ${flushErr.message}`, 'warn');
2140
- }
2141
- }
2142
- if (isCredentialOutage && pendingFillKeysForCurrentCycle.size === 0) {
2143
- this._credentialRecoveryNeeded = true;
2144
- this._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${err.message}`);
2145
- }
2146
- this._log(`Error processing fills: ${err.message}`, 'error');
2147
- if (err.stack)
2148
- this._log(err.stack, 'error');
2149
- }
2150
- // Post-processing: If new fills arrived while processing, schedule another cycle
2151
- // SAFE: Done outside lock context, no async work in finally block
2152
- if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
2153
- this._scheduleFillConsumerRestart(chainOrders);
2154
- }
648
+ return DexbotFillRuntime.consumeFillQueue(this, chainOrders);
2155
649
  }
2156
650
  /**
2157
651
  * Process fills during bootstrap phase using the standard fill pipeline.
@@ -2171,98 +665,7 @@ class DEXBot {
2171
665
  * @returns {Promise<void>}
2172
666
  */
2173
667
  async _processFillsWithBootstrapMode(chainOrders) {
2174
- if (this._incomingFillQueue.length === 0)
2175
- return;
2176
- const startTime = Date.now();
2177
- const fills = this._incomingFillQueue.splice(0);
2178
- const validFills = [];
2179
- const processedFillKeys = new Set();
2180
- let requiresOpenOrdersSync = false;
2181
- // 1. Validate and deduplicate fills
2182
- for (const fill of fills) {
2183
- if (!fill || fill.op?.[0] !== 4)
2184
- continue;
2185
- const fillOp = fill.op[1];
2186
- const gridOrder = this.manager.orders.get(fillOp.order_id) ||
2187
- Array.from(this.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
2188
- if (!gridOrder) {
2189
- // CRITICAL FIX: Even if order not in grid, we must still credit the fill proceeds
2190
- // This can happen when fills arrive after an order was marked VIRTUAL during sequential processing
2191
- let orphanFillKey = buildFillKey(fill);
2192
- if (!orphanFillKey) {
2193
- orphanFillKey = this._buildOrphanFillFallbackKey(fill);
2194
- }
2195
- if (orphanFillKey && !this._isNewFillKey(orphanFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
2196
- continue;
2197
- }
2198
- this.manager.logger.log(`[BOOTSTRAP] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
2199
- const accountingResult = await this._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
2200
- context: 'BOOTSTRAP'
2201
- });
2202
- if (accountingResult.status === 'missing_key') {
2203
- requiresOpenOrdersSync = true;
2204
- }
2205
- continue;
2206
- }
2207
- const trackedFillKey = buildFillKey(fill);
2208
- if (trackedFillKey && !this._isNewFillKey(trackedFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
2209
- continue;
2210
- }
2211
- this.manager.lockOrders([gridOrder.id]);
2212
- try {
2213
- const accountingResult = await this._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
2214
- context: 'BOOTSTRAP',
2215
- replayMessage: (op) => `[BOOTSTRAP] Replay detected for ${op.order_id}; skipping duplicate bootstrap rebalance`
2216
- });
2217
- if (accountingResult.status === 'missing_key') {
2218
- requiresOpenOrdersSync = true;
2219
- continue;
2220
- }
2221
- if (accountingResult.status !== 'applied') {
2222
- continue;
2223
- }
2224
- validFills.push({ ...fill, gridOrder });
2225
- const fillType = gridOrder.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL';
2226
- this._log(`[BOOTSTRAP] Fill detected: ${fillType} order (${fillOp.is_maker !== false ? 'maker' : 'taker'})`);
2227
- }
2228
- finally {
2229
- this.manager.unlockOrders([gridOrder.id]);
2230
- }
2231
- }
2232
- if (requiresOpenOrdersSync) {
2233
- this._log('[BOOTSTRAP] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
2234
- const bootstrapChainOpenOrders = await chainOrders.readOpenOrders(this.accountId);
2235
- const syncResult = await this.manager.syncFromOpenOrders(bootstrapChainOpenOrders);
2236
- if (syncResult.filledOrders?.length > 0) {
2237
- const queuedOrderIds = new Set(validFills.map(fill => fill?.gridOrder?.orderId).filter(Boolean));
2238
- for (const filledOrder of syncResult.filledOrders) {
2239
- if (!filledOrder?.orderId || queuedOrderIds.has(filledOrder.orderId))
2240
- continue;
2241
- validFills.push({ gridOrder: filledOrder });
2242
- queuedOrderIds.add(filledOrder.orderId);
2243
- }
2244
- }
2245
- }
2246
- await this._flushProcessedFillPersistence('bootstrap-batch');
2247
- if (validFills.length === 0)
2248
- return;
2249
- // 2. Process fills through the standard fill pipeline.
2250
- // This handles same-side replacement, symmetric rebalance, and dust cleanup
2251
- // via the same path used by the post-reset flow.
2252
- try {
2253
- this._log(`[BOOTSTRAP] Processing ${validFills.length} fill(s) through standard pipeline`, 'info');
2254
- const filledOrders = validFills.map(f => f.gridOrder);
2255
- const result = await this._processFillsWithBatching(filledOrders, new Set(), '[BOOTSTRAP] fill processing');
2256
- if (result.aborted) {
2257
- this._warn('[BOOTSTRAP] Aborted batch due to illegal state; skipping grid persistence this cycle');
2258
- }
2259
- this._metrics.fillsProcessed += validFills.length;
2260
- this._metrics.fillProcessingTimeMs += Date.now() - startTime;
2261
- }
2262
- catch (err) {
2263
- this._warn(`[BOOTSTRAP] Error processing fills: ${err.message}`);
2264
- this.manager.logger.log(`[BOOTSTRAP] Fill error: ${err.message}`, 'error');
2265
- }
668
+ return DexbotFillRuntime.processFillsWithBootstrapMode(this, chainOrders);
2266
669
  }
2267
670
  /**
2268
671
  * Set up account identifier and configure global context.
@@ -2332,43 +735,7 @@ class DEXBot {
2332
735
  * @returns {Promise<void>}
2333
736
  */
2334
737
  async placeInitialOrders() {
2335
- if (!this.manager) {
2336
- const mgrLogFile = this.config?.name ? path.join(PATHS.LOGS_DIR, `${this.config.name}.log`) : undefined;
2337
- this.manager = new OrderManager({ ...this.config, logFile: mgrLogFile });
2338
- this.manager.accountOrders = this.accountOrders;
2339
- }
2340
- this._wireStructuralGridResyncRequest();
2341
- this.manager.startBootstrap();
2342
- try {
2343
- try {
2344
- const botFunds = this.config && this.config.botFunds ? this.config.botFunds : {};
2345
- const needsPercent = (v) => typeof v === 'string' && v.includes('%');
2346
- if ((needsPercent(botFunds.buy) || needsPercent(botFunds.sell)) && (this.accountId || this.account)) {
2347
- if (typeof this.manager._fetchAccountBalancesAndSetTotals === 'function') {
2348
- await this.manager._fetchAccountBalancesAndSetTotals();
2349
- }
2350
- }
2351
- }
2352
- catch (errFetch) {
2353
- this._warn(`Could not fetch account totals before initializing grid: ${errFetch && errFetch.message ? errFetch.message : errFetch}`);
2354
- }
2355
- await Grid.initializeGrid(this.manager);
2356
- if (this.config.dryRun) {
2357
- this.manager.logger.log('Dry run enabled, skipping on-chain order placement.', 'info');
2358
- await this.manager.persistGrid();
2359
- return;
2360
- }
2361
- this.manager.logger.log('Placing initial orders on-chain...', 'info');
2362
- const ordersToActivate = this.manager.getInitialOrdersToActivate();
2363
- const orderGroups = this._buildOutsideInPairGroupsForOrders(ordersToActivate);
2364
- for (const group of orderGroups) {
2365
- await this.updateOrdersOnChainPlan({ ordersToPlace: group });
2366
- }
2367
- await this.manager.persistGrid();
2368
- }
2369
- finally {
2370
- this.manager.finishBootstrap();
2371
- }
738
+ return DexbotStartupRuntime.placeInitialOrdersImpl(this);
2372
739
  }
2373
740
  /**
2374
741
  * Build outside-in pair groups for initial order placement.
@@ -2376,11 +743,7 @@ class DEXBot {
2376
743
  * @returns {Array<Array<Object>>} Grouped order arrays
2377
744
  */
2378
745
  _buildOutsideInPairGroupsForOrders(orders) {
2379
- return buildOutsideInPairGroups(orders, {
2380
- isValid: Boolean,
2381
- getType: o => o.type,
2382
- getPrice: o => o.price,
2383
- });
746
+ return cowRuntime.buildOutsideInPairGroupsForOrders(orders);
2384
747
  }
2385
748
  /**
2386
749
  * Build outside-in pair groups for create entry contexts.
@@ -2388,11 +751,7 @@ class DEXBot {
2388
751
  * @returns {Array<Array<Object>>} Grouped entry arrays
2389
752
  */
2390
753
  _buildOutsideInPairGroupsForCreateEntries(createEntries) {
2391
- return buildOutsideInPairGroups(createEntries, {
2392
- isValid: e => Boolean(e?.context?.order),
2393
- getType: e => e.context.order.type,
2394
- getPrice: e => e.context.order.price,
2395
- });
754
+ return cowRuntime.buildOutsideInPairGroupsForCreateEntries(createEntries);
2396
755
  }
2397
756
  /**
2398
757
  * Resolve the centralized fill batch cap.
@@ -2408,19 +767,7 @@ class DEXBot {
2408
767
  * @returns {Array} Array of operation result entries
2409
768
  */
2410
769
  _extractOperationResults(result, warnContext = '') {
2411
- const extracted = extractBatchOperationResults(result);
2412
- if (Array.isArray(extracted))
2413
- return extracted;
2414
- if (result) {
2415
- const resultType = Array.isArray(result) ? 'array' : typeof result;
2416
- const keySummary = (resultType === 'object' && !Array.isArray(result))
2417
- ? Object.keys(result).slice(0, 8).join(',')
2418
- : '';
2419
- const contextSuffix = warnContext ? ` (${warnContext})` : '';
2420
- const keysSuffix = keySummary ? `; keys=[${keySummary}]` : '';
2421
- this.manager?.logger?.log(`[COW] Unrecognized operation_results shape${contextSuffix}; defaulting to empty results. resultType=${resultType}${keysSuffix}`, 'warn');
2422
- }
2423
- return [];
770
+ return cowRuntime.extractOperationResults(result, warnContext, this.manager?.logger?.log?.bind(this.manager?.logger));
2424
771
  }
2425
772
  /**
2426
773
  * Find CREATE operation contexts whose broadcast result did not include a chain order id.
@@ -2430,19 +777,7 @@ class DEXBot {
2430
777
  * @returns {Array<{index:number, ctx:Object}>} Missing create result contexts.
2431
778
  */
2432
779
  _findMissingCreateResultContexts(operationResults, opContexts) {
2433
- const missing = [];
2434
- if (!Array.isArray(opContexts))
2435
- return missing;
2436
- for (let i = 0; i < opContexts.length; i++) {
2437
- const ctx = opContexts[i];
2438
- if (ctx?.kind !== 'create')
2439
- continue;
2440
- const chainOrderId = operationResults?.[i]?.[1];
2441
- if (!chainOrderId || !/^1\.7\.\d+$/.test(String(chainOrderId))) {
2442
- missing.push({ index: i, ctx });
2443
- }
2444
- }
2445
- return missing;
780
+ return cowRuntime.findMissingCreateResultContexts(operationResults, opContexts);
2446
781
  }
2447
782
  /**
2448
783
  * Run an immediate chain sync after a successful CREATE broadcast returned incomplete ids.
@@ -2456,42 +791,7 @@ class DEXBot {
2456
791
  * @returns {Promise<void>}
2457
792
  */
2458
793
  async _recoverAfterMissingCreateResults(reason = 'missing create operation results') {
2459
- try {
2460
- const accountRef = this.accountId || this.account?.id || this.account;
2461
- if (!accountRef || !this.manager || !chainOrders?.readOpenOrders) {
2462
- this.manager?.logger?.log?.(`[COW] Recovery sync unavailable after ${reason}`, 'warn');
2463
- return;
2464
- }
2465
- const preRecoveryMissingCreateBlockers = Array.isArray(this.manager._lastUnmatchedChainOrders)
2466
- ? this.manager._lastUnmatchedChainOrders
2467
- .filter(order => order?.reason === 'missing-create-result')
2468
- .map(order => ({ ...order }))
2469
- : [];
2470
- const openOrders = await chainOrders.readOpenOrders(accountRef);
2471
- const recoveryResult = await this.manager.syncFromOpenOrders(openOrders, {
2472
- skipAccounting: false,
2473
- });
2474
- this._preserveMissingCreateBlockersAfterRecovery(preRecoveryMissingCreateBlockers, recoveryResult);
2475
- // Persist any master grid mutations from the recovery sync. The
2476
- // caller returned from the COW catch handler before reaching the
2477
- // success-path persistGrid.
2478
- if (typeof this.manager.persistGrid === 'function') {
2479
- await this.manager.persistGrid();
2480
- }
2481
- }
2482
- catch (err) {
2483
- this.manager?.logger?.log?.(`[COW] CRITICAL: Recovery sync failed after ${reason}: ${err.message}`, 'error');
2484
- if (typeof this.manager?.requestStructuralGridResync === 'function') {
2485
- try {
2486
- await this.manager.requestStructuralGridResync(`recovery sync failed after ${reason}`, {
2487
- error: err.message
2488
- });
2489
- }
2490
- catch (scheduleErr) {
2491
- this.manager?.logger?.log?.(`[COW] CRITICAL: Failed to schedule structural resync after recovery failure: ${scheduleErr.message}`, 'error');
2492
- }
2493
- }
2494
- }
794
+ return cowRuntime.recoverAfterMissingCreateResults(this, reason);
2495
795
  }
2496
796
  /**
2497
797
  * Restore unresolved missing-create blockers after recovery if sync did not adopt them.
@@ -2501,30 +801,7 @@ class DEXBot {
2501
801
  * @returns {void}
2502
802
  */
2503
803
  _preserveMissingCreateBlockersAfterRecovery(blockers, recoveryResult) {
2504
- if (!Array.isArray(blockers) || blockers.length === 0 || !this.manager)
2505
- return;
2506
- const adoptedSlotIds = new Set((Array.isArray(recoveryResult?.updatedOrders) ? recoveryResult.updatedOrders : [])
2507
- .filter(order => order?.id && order?.orderId)
2508
- .map(order => order.id));
2509
- const unresolvedBlockers = blockers.filter(blocker => !blocker.slotId || !adoptedSlotIds.has(blocker.slotId));
2510
- if (unresolvedBlockers.length === 0)
2511
- return;
2512
- const currentUnmatched = Array.isArray(this.manager._lastUnmatchedChainOrders)
2513
- ? this.manager._lastUnmatchedChainOrders
2514
- : [];
2515
- const currentKeys = new Set(currentUnmatched.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
2516
- const restored = [...currentUnmatched];
2517
- for (const blocker of unresolvedBlockers) {
2518
- const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
2519
- if (!currentKeys.has(key))
2520
- restored.push({ ...blocker });
2521
- }
2522
- if (restored.length !== currentUnmatched.length) {
2523
- this.manager._lastUnmatchedChainOrders = restored;
2524
- this.manager._lastUnmatchedChainOrdersAt = Date.now();
2525
- this.manager.logger?.log?.(`[COW] Preserving ${restored.length - currentUnmatched.length} missing-create blocker(s) after recovery sync; ` +
2526
- `chain snapshot did not account for the affected slot(s).`, 'warn');
2527
- }
804
+ return cowRuntime.preserveMissingCreateBlockersAfterRecovery(this, blockers, recoveryResult);
2528
805
  }
2529
806
  /**
2530
807
  * Merge missing CREATE result contexts into manager._lastUnmatchedChainOrders.
@@ -2538,42 +815,7 @@ class DEXBot {
2538
815
  * @returns {void}
2539
816
  */
2540
817
  _markMissingCreateResultsAsStructuralBlocker(missingCreateResults) {
2541
- const blockers = Array.isArray(missingCreateResults)
2542
- ? missingCreateResults.map(item => {
2543
- const order = item.ctx?.order || {};
2544
- const fingerprint = [
2545
- `type=${order.type || 'unknown'}`,
2546
- `price=${Format.formatPrice6(order.price)}`,
2547
- `size=${Format.formatAmount(order.size)}`
2548
- ].join(',');
2549
- return {
2550
- chainOrderId: 'unknown',
2551
- type: order.type || null,
2552
- price: order.price,
2553
- size: order.size,
2554
- slotId: order.id || item.ctx?.id || null,
2555
- reason: 'missing-create-result',
2556
- operationIndex: item.index,
2557
- fingerprint,
2558
- };
2559
- })
2560
- : [];
2561
- if (this.manager && blockers.length > 0) {
2562
- const existing = Array.isArray(this.manager._lastUnmatchedChainOrders)
2563
- ? this.manager._lastUnmatchedChainOrders
2564
- : [];
2565
- const keys = new Set(existing.map(order => `${order.reason || ''}:${order.slotId || ''}:${order.operationIndex ?? ''}`));
2566
- const merged = [...existing];
2567
- for (const blocker of blockers) {
2568
- const key = `${blocker.reason || ''}:${blocker.slotId || ''}:${blocker.operationIndex ?? ''}`;
2569
- if (!keys.has(key)) {
2570
- merged.push(blocker);
2571
- keys.add(key);
2572
- }
2573
- }
2574
- this.manager._lastUnmatchedChainOrders = merged;
2575
- this.manager._lastUnmatchedChainOrdersAt = Date.now();
2576
- }
818
+ return cowRuntime.markMissingCreateResultsAsStructuralBlocker(this, missingCreateResults);
2577
819
  }
2578
820
  /**
2579
821
  * Format an unmatched chain order/blocker for COW logs.
@@ -2582,7 +824,7 @@ class DEXBot {
2582
824
  * @returns {string} Compact human-readable diagnostic.
2583
825
  */
2584
826
  _formatUnmatchedChainOrderForLog(order) {
2585
- return formatUnmatchedChainOrder(order);
827
+ return cowRuntime.formatUnmatchedChainOrderForLog(order);
2586
828
  }
2587
829
  /**
2588
830
  * Record a pending CREATE broadcast on the manager.
@@ -2605,35 +847,7 @@ class DEXBot {
2605
847
  * @returns {void}
2606
848
  */
2607
849
  _recordPendingBroadcast(entry) {
2608
- if (!this.manager || !entry || !entry.order)
2609
- return;
2610
- if (!this.manager._pendingBroadcasts || !(this.manager._pendingBroadcasts instanceof Map)) {
2611
- this.manager._pendingBroadcasts = new Map();
2612
- }
2613
- const fingerprint = buildCreateOpFingerprint({
2614
- side: entry.order.type,
2615
- assetA: this.manager?.assets?.assetA?.id,
2616
- assetB: this.manager?.assets?.assetB?.id,
2617
- sellInt: entry.finalInts?.sell,
2618
- receiveInt: entry.finalInts?.receive,
2619
- slotId: entry.order.id
2620
- });
2621
- if (!fingerprint) {
2622
- this.manager.logger.log?.(`[COW] Skipped pending-broadcast record: could not build fingerprint for ${entry.order?.id || 'unknown'}`, 'warn');
2623
- return;
2624
- }
2625
- this.manager._pendingBroadcasts.set(fingerprint, {
2626
- fingerprint,
2627
- opIndex: entry.opIndex,
2628
- ctxIndex: entry.ctxIndex,
2629
- slotId: entry.order.id,
2630
- orderId: entry.order.id,
2631
- orderType: entry.order.type,
2632
- order: entry.order,
2633
- finalInts: entry.finalInts,
2634
- batchId: this._currentBatchId || null,
2635
- recordedAt: Date.now()
2636
- });
850
+ return cowRuntime.recordPendingBroadcast(this, entry);
2637
851
  }
2638
852
  /**
2639
853
  * Clear the pending-broadcast cache.
@@ -2644,9 +858,7 @@ class DEXBot {
2644
858
  * _reconcileAfterUncertainBroadcast before calling this).
2645
859
  */
2646
860
  _clearPendingBroadcasts() {
2647
- if (this.manager && this.manager._pendingBroadcasts instanceof Map) {
2648
- this.manager._pendingBroadcasts.clear();
2649
- }
861
+ return cowRuntime.clearPendingBroadcasts(this.manager?._pendingBroadcasts);
2650
862
  }
2651
863
  /**
2652
864
  * Build a fingerprint for an on-chain order so it can be matched against
@@ -2657,19 +869,7 @@ class DEXBot {
2657
869
  * @returns {string|null} Fingerprint or null on bad input
2658
870
  */
2659
871
  _buildChainOrderFingerprint(chainOrder, slotId) {
2660
- if (!chainOrder || !slotId)
2661
- return null;
2662
- const normalized = this._normalizeChainOrderForPendingMatch(chainOrder);
2663
- if (!normalized)
2664
- return null;
2665
- return buildCreateOpFingerprint({
2666
- side: normalized.side,
2667
- assetA: normalized.assetA,
2668
- assetB: normalized.assetB,
2669
- sellInt: normalized.sellInt,
2670
- receiveInt: normalized.receiveInt,
2671
- slotId
2672
- });
872
+ return cowRuntime.buildChainOrderFingerprint(this, chainOrder, slotId);
2673
873
  }
2674
874
  /**
2675
875
  * Normalize raw BitShares limit_order_object data into the integer tuple
@@ -2683,41 +883,7 @@ class DEXBot {
2683
883
  * @returns {{side: string, assetA: string, assetB: string, sellInt: number, receiveInt: number}|null}
2684
884
  */
2685
885
  _normalizeChainOrderForPendingMatch(chainOrder) {
2686
- if (!chainOrder)
2687
- return null;
2688
- const assetA = this.manager?.assets?.assetA?.id;
2689
- const assetB = this.manager?.assets?.assetB?.id;
2690
- if (!assetA || !assetB)
2691
- return null;
2692
- const explicitSide = (chainOrder.type === 'buy' || chainOrder.type === 'sell')
2693
- ? chainOrder.type
2694
- : null;
2695
- const explicitSell = chainOrder.sellInt ?? chainOrder.sell;
2696
- const explicitReceive = chainOrder.receiveInt ?? chainOrder.receive;
2697
- if (explicitSide && Number.isFinite(Number(explicitSell)) && Number.isFinite(Number(explicitReceive))) {
2698
- return {
2699
- side: explicitSide,
2700
- assetA,
2701
- assetB,
2702
- sellInt: Number(explicitSell),
2703
- receiveInt: Number(explicitReceive)
2704
- };
2705
- }
2706
- const base = chainOrder.sell_price?.base;
2707
- const quote = chainOrder.sell_price?.quote;
2708
- if (!base || !quote || !base.asset_id || !quote.asset_id)
2709
- return null;
2710
- const baseAmount = Number(base.amount);
2711
- const quoteAmount = Number(quote.amount);
2712
- if (!Number.isFinite(baseAmount) || !Number.isFinite(quoteAmount))
2713
- return null;
2714
- if (base.asset_id === assetA && quote.asset_id === assetB) {
2715
- return { side: 'sell', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
2716
- }
2717
- if (base.asset_id === assetB && quote.asset_id === assetA) {
2718
- return { side: 'buy', assetA, assetB, sellInt: baseAmount, receiveInt: quoteAmount };
2719
- }
2720
- return null;
886
+ return cowRuntime.normalizeChainOrderForPendingMatch(this, chainOrder);
2721
887
  }
2722
888
  /**
2723
889
  * Find a chain order that matches a planned slot using price+size proximity.
@@ -2735,56 +901,7 @@ class DEXBot {
2735
901
  * @returns {Object|null} Matching chain order, or null
2736
902
  */
2737
903
  _findChainOrderForSlot(chainOrders, slotId, planned) {
2738
- if (!Array.isArray(chainOrders) || !slotId)
2739
- return null;
2740
- const assetA = this.manager?.assets?.assetA?.id;
2741
- const assetB = this.manager?.assets?.assetB?.id;
2742
- if (!assetA || !assetB)
2743
- return null;
2744
- // 1. Exact fingerprint match.
2745
- for (const o of chainOrders) {
2746
- const fp = this._buildChainOrderFingerprint(o, slotId);
2747
- if (fp && this.manager._pendingBroadcasts?.has(fp)) {
2748
- return o;
2749
- }
2750
- }
2751
- if (!planned || !Number.isFinite(Number(planned.sell)) || !Number.isFinite(Number(planned.receive))) {
2752
- return null;
2753
- }
2754
- // 2. Near match: same side, sell int within 1, receive int within 1% or 2 units.
2755
- const targetSell = Number(planned.sell);
2756
- const targetReceive = Number(planned.receive);
2757
- const plannedSide = planned.orderType ||
2758
- planned.side ||
2759
- this.manager._pendingBroadcasts?.get?.(planned.fingerprint)?.orderType ||
2760
- this.manager.orders.get(slotId)?.type;
2761
- if (plannedSide !== 'buy' && plannedSide !== 'sell') {
2762
- return null;
2763
- }
2764
- let best = null;
2765
- let bestDistance = Infinity;
2766
- for (const o of chainOrders) {
2767
- const normalized = this._normalizeChainOrderForPendingMatch(o);
2768
- if (!normalized)
2769
- continue;
2770
- if (normalized.side !== plannedSide)
2771
- continue;
2772
- const sell = Number(normalized.sellInt);
2773
- const receive = Number(normalized.receiveInt);
2774
- if (!Number.isFinite(sell) || !Number.isFinite(receive))
2775
- continue;
2776
- const sellDelta = Math.abs(sell - targetSell);
2777
- const receiveDelta = Math.abs(receive - targetReceive);
2778
- const receiveTol = Math.max(2, Math.floor(targetReceive * 0.01));
2779
- if (sellDelta > 1 || receiveDelta > receiveTol)
2780
- continue;
2781
- const distance = sellDelta * 1000 + receiveDelta;
2782
- if (distance < bestDistance) {
2783
- best = o;
2784
- bestDistance = distance;
2785
- }
2786
- }
2787
- return best;
904
+ return cowRuntime.findChainOrderForSlot(this, chainOrders, slotId, planned);
2788
905
  }
2789
906
  /**
2790
907
  * Reconcile a broadcast whose chain state is unknown.
@@ -2799,287 +916,24 @@ class DEXBot {
2799
916
  * 2. For each pending-broadcast entry (fingerprinted CREATE op), look
2800
917
  * for a matching chain order. If found: adopt it (set the opContext's
2801
918
  * chainOrderId and continue with the existing planned slot).
2802
- * 3. For pending entries with no chain match: virtualize (mark the
2803
- * opContext as discarded; the planned slot stays empty until the
2804
- * next planning cycle).
2805
- * 4. Persist + log a structured [COW][UNCERTAIN] summary.
2806
- *
2807
- * After this method returns, the bot has either adopted the on-chain
2808
- * result (good case — chain accepted but we didn't see the reply) or
2809
- * accepted the discard (chain rejected or never received the op).
2810
- *
2811
- * @param {BroadcastUncertainError} err - The thrown error
2812
- * @param {Array<Object>} opContexts - Original opContexts from the failed batch
2813
- * @returns {Promise<Object>} Result object compatible with batch return shape
2814
- */
2815
- async _reconcileAfterUncertainBroadcast(err, opContexts, options = {}) {
2816
- if (this.manager?._fillProcessingLock &&
2817
- typeof this.manager._fillProcessingLock.acquire === 'function' &&
2818
- !this.manager._fillProcessingLock.isReentrant()) {
2819
- return this.manager._fillProcessingLock.acquire(() => this._reconcileAfterUncertainBroadcast(err, opContexts, options));
2820
- }
2821
- return this._reconcileAfterUncertainBroadcastImpl(err, opContexts, options);
2822
- }
2823
- async _reconcileAfterUncertainBroadcastImpl(err, opContexts, options) {
2824
- const startedAt = Date.now();
2825
- const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
2826
- ? Array.from(this.manager._pendingBroadcasts.values())
2827
- : [];
2828
- const createContextCount = opContexts.filter(c => c && c.kind === 'create').length;
2829
- const nonCreateContextCount = opContexts.length - createContextCount;
2830
- this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
2831
- `creates=${createContextCount} nonCreates=${nonCreateContextCount} ` +
2832
- `staleSinceMs=${err?.timeoutMs || 'n/a'}. Entering reconcile-then-decide.`, 'warn');
2833
- if (!chainOrders?.readOpenOrders) {
2834
- this.manager.logger.log('[COW][UNCERTAIN] readOpenOrders unavailable; falling back to structural resync only.', 'error');
2835
- if (typeof this.manager.requestStructuralGridResync === 'function') {
2836
- await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders unavailable', { batchId: err?.batchId || null });
2837
- }
2838
- this._clearPendingBroadcasts();
2839
- return { executed: false, hadRotation: false, uncertain: true };
2840
- }
2841
- // 1. Read the chain
2842
- const accountRef = this.accountId || this.account?.id || this.account;
2843
- let chainSnapshot = [];
2844
- try {
2845
- chainSnapshot = await chainOrders.readOpenOrders(accountRef);
2846
- }
2847
- catch (readErr) {
2848
- this.manager.logger.log(`[COW][UNCERTAIN] readOpenOrders failed: ${readErr?.message || readErr}. ` +
2849
- `Falling back to structural resync.`, 'error');
2850
- if (typeof this.manager.requestStructuralGridResync === 'function') {
2851
- await this.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders failed', { batchId: err?.batchId || null, error: readErr?.message || String(readErr) });
2852
- }
2853
- this._clearPendingBroadcasts();
2854
- return { executed: false, hadRotation: false, uncertain: true };
2855
- }
2856
- const adopted = [];
2857
- let discarded = [];
2858
- // 2. For each pending broadcast, look for a chain match.
2859
- for (const entry of pending) {
2860
- const match = this._findChainOrderForSlot(chainSnapshot, entry.slotId, {
2861
- sell: entry.finalInts?.sell,
2862
- receive: entry.finalInts?.receive,
2863
- orderType: entry.orderType || entry.order?.type,
2864
- fingerprint: entry.fingerprint
2865
- });
2866
- if (match) {
2867
- adopted.push({
2868
- slotId: entry.slotId,
2869
- chainOrderId: match.id,
2870
- orderType: entry.orderType || entry.order?.type,
2871
- });
2872
- this.manager._pendingBroadcasts.delete(entry.fingerprint);
2873
- }
2874
- else {
2875
- // Preserve full entry context so the recheck below can use
2876
- // both the fingerprint match path (against _pendingBroadcasts)
2877
- // and the near-match path (via finalInts/orderType), not just
2878
- // slotId alone.
2879
- discarded.push(entry);
2880
- }
2881
- }
2882
- // ---- RACE MITIGATION: block-delay recheck before discarding creates ----
2883
- // BROADCAST_DEADLINE fires while the transaction may still be valid
2884
- // and land in the next block. Wait for up to 3 blocks (~1 BitShares
2885
- // block interval each) and re-check before permanently discarding.
2886
- const UNCERTAIN_RECHECK_MAX_ATTEMPTS = this.config.pipelineTiming?.RETRY_MAX_ATTEMPTS ?? PIPELINE_TIMING.RETRY_MAX_ATTEMPTS; // 3
2887
- const UNCERTAIN_RECHECK_INTERVAL_MS = TIMING.MILLISECONDS_PER_SECOND * 3; // 3s
2888
- let recheckRound = 0;
2889
- while (discarded.length > 0 && recheckRound < UNCERTAIN_RECHECK_MAX_ATTEMPTS) {
2890
- recheckRound++;
2891
- await new Promise(r => setTimeout(r, UNCERTAIN_RECHECK_INTERVAL_MS));
2892
- let freshSnapshot;
2893
- try {
2894
- freshSnapshot = await chainOrders.readOpenOrders(accountRef);
2895
- }
2896
- catch {
2897
- break;
2898
- }
2899
- const stillDiscarded = [];
2900
- for (const entry of discarded) {
2901
- const match = this._findChainOrderForSlot(freshSnapshot, entry.slotId, {
2902
- sell: entry.finalInts?.sell,
2903
- receive: entry.finalInts?.receive,
2904
- orderType: entry.orderType || entry.order?.type,
2905
- fingerprint: entry.fingerprint
2906
- });
2907
- if (match) {
2908
- adopted.push({
2909
- slotId: entry.slotId,
2910
- chainOrderId: match.id,
2911
- orderType: entry.orderType || entry.order?.type,
2912
- });
2913
- this.manager.logger.log(`[COW][UNCERTAIN] CREATE re-adopted after ${recheckRound} block(s): ` +
2914
- `${entry.slotId}->${match.id}`, 'info');
2915
- }
2916
- else {
2917
- stillDiscarded.push(entry);
2918
- }
2919
- }
2920
- discarded = stillDiscarded;
2921
- }
2922
- // 3. Re-read chain with latest state. The initial snapshot may be
2923
- // stale for non-CREATE ops (updates/cancels) not tracked in
2924
- // _pendingBroadcasts — the seed cause of the slot-115 cascade.
2925
- let latestSnapshot;
2926
- try {
2927
- latestSnapshot = await chainOrders.readOpenOrders(accountRef);
2928
- }
2929
- catch {
2930
- latestSnapshot = chainSnapshot;
2931
- }
2932
- // Always sync to link adopted chain orders into grid slots.
2933
- // Otherwise an adopted slot stays VIRTUAL and the next COW cycle
2934
- // places a duplicate order at the same price.
2935
- let hadRotation = false;
2936
- if (latestSnapshot && latestSnapshot.length > 0 && this.manager?.syncFromOpenOrders) {
2937
- try {
2938
- await this.manager.syncFromOpenOrders(latestSnapshot, {
2939
- skipAccounting: true,
2940
- });
2941
- hadRotation = true;
2942
- }
2943
- catch (syncErr) {
2944
- this.manager.logger.log(`[COW][UNCERTAIN] syncFromOpenOrders failed during recovery: ${syncErr?.message || syncErr}`, 'error');
2945
- // Fallback: reload from persisted snapshot + re-sync.
2946
- this.manager.logger.log('[COW][UNCERTAIN] syncFromOpenOrders failed; falling back to full grid reload from persisted snapshot', 'warn');
2947
- const fallbackResult = await this._recoverFromPersistedGrid();
2948
- if (fallbackResult.success) {
2949
- hadRotation = true;
2950
- }
2951
- else {
2952
- this.manager.logger.log(`[COW][UNCERTAIN] Persisted grid reload failed: ${fallbackResult.reason}. ` +
2953
- `Escalating to structural resync.`, 'warn');
2954
- if (typeof this.manager.requestStructuralGridResync === 'function') {
2955
- this.manager.requestStructuralGridResync('cow-uncertain-recovery-failed', { reason: fallbackResult.reason }).catch((escalateErr) => {
2956
- this.manager.logger.log(`[COW][UNCERTAIN] Escalation to structural resync failed: ${escalateErr.message}`, 'error');
2957
- });
2958
- }
2959
- }
2960
- }
2961
- }
2962
- // 4. Log structured summary.
2963
- const elapsedMs = Date.now() - startedAt;
2964
- const heartbeatAgeMs = this._lastBroadcastHeartbeatAt
2965
- ? Date.now() - this._lastBroadcastHeartbeatAt
2966
- : null;
2967
- this.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
2968
- `staleSinceMs=${err?.timeoutMs || 'n/a'} heartbeatAgeMs=${heartbeatAgeMs ?? 'n/a'} ` +
2969
- `adopted=${adopted.length} discarded=${discarded.length} recheckRounds=${recheckRound} elapsedMs=${elapsedMs}`, adopted.length > 0 ? 'info' : 'warn');
2970
- if (adopted.length > 0) {
2971
- this.manager.logger.log(`[COW][UNCERTAIN] Adopted chain orders: ${adopted
2972
- .map(a => `${a.slotId}->${a.chainOrderId}`)
2973
- .join(', ')}`, 'info');
2974
- }
2975
- if (discarded.length > 0) {
2976
- this.manager.logger.log(`[COW][UNCERTAIN] Discarded planned CREATEs (no chain match after ${recheckRound} recheck(s)): ${discarded
2977
- .map(d => d.slotId)
2978
- .join(', ')}`, 'warn');
2979
- // Restore target grid sizes for discarded CREATEs so the slots are
2980
- // not left as virtual/0 after recovery. Without this, the slot stays
2981
- // at size 0 (set by the fill handler) and is never reactivated until
2982
- // a fresh fill cycle triggers another rebalance that happens to succeed.
2983
- // entry.order is the pending-broadcast target order (captured at broadcast
2984
- // time); its .id matches entry.slotId — the lookup below keys on slotId.
2985
- for (const entry of discarded) {
2986
- if (entry.order && entry.slotId) {
2987
- const current = this.manager.orders.get(entry.slotId);
2988
- if (current && current.state === ORDER_STATES.VIRTUAL && !current.orderId) {
2989
- try {
2990
- await this.manager._applyOrderUpdate({ ...entry.order, state: ORDER_STATES.VIRTUAL, orderId: null }, 'uncertain-recovery-restore-size', { skipAccounting: true, fee: 0 });
2991
- this.manager.logger.log(`[COW][UNCERTAIN] Restored target size for discarded CREATE slot ${entry.slotId} (size: ${entry.order.size})`, 'info');
2992
- }
2993
- catch (restoreErr) {
2994
- this.manager.logger.log(`[COW][UNCERTAIN] Failed to restore size for slot ${entry.slotId}: ${restoreErr?.message || restoreErr}`, 'warn');
2995
- }
2996
- }
2997
- }
2998
- }
2999
- }
3000
- // ---- Structural resync safeguard after skipAccounting restore ----
3001
- // The discarded CREATE restore above used skipAccounting: true to avoid
3002
- // double-counting when the structural resync recalculates accounting
3003
- // from scratch. Ensure one is scheduled — if already in-flight (timer
3004
- // or running), the existing resync will handle the accounting fix.
3005
- if (discarded.length > 0 && typeof this.manager?.requestStructuralGridResync === 'function') {
3006
- const alreadyScheduled = this._structuralGridResyncRunning || this._structuralGridResyncTimer;
3007
- if (!alreadyScheduled) {
3008
- this.manager.logger.log(`[COW][UNCERTAIN] Scheduling structural resync after discarded CREATE restore ` +
3009
- `(skipAccounting used — resync needed for fund recalculation)`, 'info');
3010
- this.manager.requestStructuralGridResync('cow-uncertain-accounting-repair', { discardedCount: discarded.length }).catch((err) => {
3011
- this.manager.logger.log(`[COW][UNCERTAIN] Failed to schedule accounting repair resync: ${err.message}`, 'error');
3012
- });
3013
- }
3014
- else {
3015
- this.manager.logger.log(`[COW][UNCERTAIN] Structural resync already ${this._structuralGridResyncRunning ? 'running' : 'scheduled'}; ` +
3016
- `it will repair accounting after discarded CREATE restore`, 'debug');
3017
- }
3018
- }
3019
- this._clearPendingBroadcasts();
3020
- // Post-recovery safety net: if there are still unmatched chain
3021
- // orders after reconcile-then-decide, cancel ONE per cycle. The cap
3022
- // is enforced inside the helper via _autoCancelOrphanCycleMarker.
3023
- try {
3024
- const autoCancelResult = await this._autoCancelOneUnmatchedOrphan();
3025
- if (autoCancelResult.cancelled) {
3026
- this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancelled orphan ${autoCancelResult.orderId} ` +
3027
- `after recovery (cap=1/cycle).`, 'info');
3028
- }
3029
- }
3030
- catch (orphanErr) {
3031
- this.manager.logger.log(`[COW][UNCERTAIN] Auto-cancel pass failed: ${orphanErr?.message || orphanErr}`, 'warn');
3032
- }
3033
- // Boundary shift recovery: the COW batch that should have committed the
3034
- // boundary shift from this fill cycle failed before commit. Only COUNT
3035
- // ADOPTED CREATEs — orders that actually landed on-chain. Discarded
3036
- // CREATEs represent orders that never existed, so the grid must NOT
3037
- // shift the boundary for them. The next fill cycle's
3038
- // calculateTargetGrid → deriveTargetBoundary will recompute the correct
3039
- // boundary from scratch for any discarded slots.
3040
- //
3041
- // NOTE: Direct mutation of manager.boundaryIdx outside a COW commit
3042
- // violates the invariant stated in grid.ts:1217-1220 (boundary must only
3043
- // be updated atomically inside _commitWorkingGrid). This is intentional
3044
- // here because the COW commit already failed — the invariant was already
3045
- // broken by the broadcast uncertainty, and the adjustment only restores
3046
- // the state to what the successful commit would have produced for
3047
- // orders that are confirmed on-chain.
3048
- if (hadRotation && this.manager && adopted.length > 0) {
3049
- let boundaryShift = 0;
3050
- for (const entry of adopted) {
3051
- if (entry.orderType === ORDER_TYPES.SELL) {
3052
- boundaryShift--; // SELL CREATE → fill was BUY → boundary LEFT
3053
- }
3054
- else if (entry.orderType === ORDER_TYPES.BUY) {
3055
- boundaryShift++; // BUY CREATE → fill was SELL → boundary RIGHT
3056
- }
3057
- }
3058
- if (boundaryShift !== 0) {
3059
- const oldIdx = this.manager.boundaryIdx;
3060
- if (typeof oldIdx === 'number') {
3061
- const maxIdx = Math.max(0, (this.manager.orders?.size || 1) - 1);
3062
- const newIdx = Math.max(0, Math.min(maxIdx, oldIdx + boundaryShift));
3063
- if (newIdx !== oldIdx) {
3064
- this.manager.boundaryIdx = newIdx;
3065
- this.manager.logger.log(`[COW][UNCERTAIN] Boundary adjusted by ${boundaryShift} ` +
3066
- `(${adopted.length} adopted CREATE(s), ${discarded.length} discarded): ` +
3067
- `${oldIdx} → ${newIdx}. Next COW cycle will generate replacement opposite-side orders.`, 'warn');
3068
- if (typeof this.manager._markGridDirty === 'function') {
3069
- this.manager._markGridDirty();
3070
- }
3071
- }
3072
- }
3073
- }
3074
- }
3075
- // Persist master grid mutations from the reconciliation sync and any
3076
- // orphan auto-cancellation that ran above. These apply outside the COW
3077
- // broadcast path and would otherwise be in-memory only. The boundary
3078
- // adjustment above is also persisted here.
3079
- if (typeof this.manager?.persistGrid === 'function') {
3080
- await this.manager.persistGrid();
3081
- }
3082
- return { executed: false, hadRotation, uncertain: true, adopted, discarded };
919
+ * 3. For pending entries with no chain match: virtualize (mark the
920
+ * opContext as discarded; the planned slot stays empty until the
921
+ * next planning cycle).
922
+ * 4. Persist + log a structured [COW][UNCERTAIN] summary.
923
+ *
924
+ * After this method returns, the bot has either adopted the on-chain
925
+ * result (good case — chain accepted but we didn't see the reply) or
926
+ * accepted the discard (chain rejected or never received the op).
927
+ *
928
+ * @param {BroadcastUncertainError} err - The thrown error
929
+ * @param {Array<Object>} opContexts - Original opContexts from the failed batch
930
+ * @returns {Promise<Object>} Result object compatible with batch return shape
931
+ */
932
+ async _reconcileAfterUncertainBroadcast(err, opContexts, options = {}) {
933
+ return cowRuntime.reconcileAfterUncertainBroadcast(this, err, opContexts, options);
934
+ }
935
+ async _reconcileAfterUncertainBroadcastImpl(err, opContexts, options) {
936
+ return cowRuntime.reconcileAfterUncertainBroadcastImpl(this, err, opContexts, options);
3083
937
  }
3084
938
  /**
3085
939
  * Auto-cancel a price-drift orphan from the unmatched-order snapshot.
@@ -3107,72 +961,7 @@ class DEXBot {
3107
961
  * @returns {Promise<{cancelled: boolean, orderId?: string, reason?: string}>}
3108
962
  */
3109
963
  async _autoCancelOneUnmatchedOrphan() {
3110
- const cycleId = this._currentCycleId || 0;
3111
- // FIX 4: Higher cap during recovery mode so orphan backlog clears faster.
3112
- const recoveryActive = this.manager?._recoveryState?.structuralResyncRequested === true;
3113
- const cycleCap = recoveryActive ? 5 : 1;
3114
- if (this._autoCancelOrphanCycleMarker === cycleId) {
3115
- if (this._autoCancelOrphanSubCount >= cycleCap) {
3116
- return { cancelled: false, reason: 'cap-reached-this-cycle', subCount: this._autoCancelOrphanSubCount };
3117
- }
3118
- }
3119
- else {
3120
- // Reset sub-count on first call this cycle
3121
- this._autoCancelOrphanCycleMarker = cycleId;
3122
- this._autoCancelOrphanSubCount = 0;
3123
- }
3124
- const pending = (this.manager && this.manager._pendingBroadcasts instanceof Map)
3125
- ? this.manager._pendingBroadcasts.size
3126
- : 0;
3127
- if (pending > 0) {
3128
- return { cancelled: false, reason: 'pending-broadcasts-active' };
3129
- }
3130
- const unmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
3131
- ? this.manager._lastUnmatchedChainOrders
3132
- : [];
3133
- if (unmatched.length === 0) {
3134
- return { cancelled: false, reason: 'no-unmatched' };
3135
- }
3136
- // Check for fingerprinted entries first — these came from a pending
3137
- // broadcast (missing-create-result) and must be handled by the recovery
3138
- // path, not by auto-cancel. The first fingerprinted entry is checked
3139
- // regardless of its reason field.
3140
- const fingerprinted = unmatched.find(u => u && u.fingerprint);
3141
- if (fingerprinted) {
3142
- return { cancelled: false, reason: 'fingerprinted-handle-via-recovery' };
3143
- }
3144
- // Only cancel price-drift orphans — these are surplus orders that
3145
- // drifted away from their assigned slot price and have no adoptable
3146
- // grid slot. All other unmatched orders (duplicate-price-level,
3147
- // already-matched-slot, etc.) are adoptable positions that the structural
3148
- // resync will integrate into the grid — cancelling them destroys capital.
3149
- const target = unmatched.find(u => u && u.reason === 'price-drift-orphan');
3150
- if (!target) {
3151
- return { cancelled: false, reason: 'no-price-drift-orphan', message: 'no price-drift orphan to cancel; other unmatched orders are adoptable' };
3152
- }
3153
- const orderId = target.id || target.orderId || target.chainOrderId;
3154
- if (!orderId) {
3155
- return { cancelled: false, reason: 'no-orderId' };
3156
- }
3157
- if (!chainOrders?.cancelOrder) {
3158
- return { cancelled: false, reason: 'cancelOrder-unavailable' };
3159
- }
3160
- try {
3161
- await chainOrders.cancelOrder(this.account, this.privateKey, orderId);
3162
- if (typeof chainOrders.recordOwnCancel === 'function') {
3163
- chainOrders.recordOwnCancel(orderId);
3164
- }
3165
- // Increment sub-count only after a confirmed successful cancel.
3166
- // Failed cancels do not consume the per-cycle cap budget.
3167
- this._autoCancelOrphanSubCount++;
3168
- this.manager.logger.log(`[COW] Auto-cancelled ${this._autoCancelOrphanSubCount}/${unmatched.length} unmatched chain order ` +
3169
- `(${this._formatUnmatchedChainOrderForLog(target)}) — per-cycle cap=${cycleCap}.`, 'warn');
3170
- return { cancelled: true, orderId };
3171
- }
3172
- catch (err) {
3173
- this.manager.logger.log(`[COW] Auto-cancel of unmatched chain order ${orderId} failed: ${err?.message || err}`, 'error');
3174
- return { cancelled: false, reason: 'cancel-failed', error: err?.message || String(err) };
3175
- }
964
+ return cowRuntime.autoCancelOneUnmatchedOrphan(this);
3176
965
  }
3177
966
  // Pair mode applies only when create contexts include both BUY and SELL.
3178
967
  // Single-side create batches intentionally remain a single executeBatch.
@@ -3182,21 +971,7 @@ class DEXBot {
3182
971
  * @returns {boolean} True if pair mode should be used
3183
972
  */
3184
973
  _shouldExecuteCreatePairMode(opContexts) {
3185
- if (!Array.isArray(opContexts) || opContexts.length < 2)
3186
- return false;
3187
- if (!opContexts.every(ctx => ctx?.kind === 'create' && ctx?.order))
3188
- return false;
3189
- let hasBuy = false;
3190
- let hasSell = false;
3191
- for (const ctx of opContexts) {
3192
- if (ctx.order.type === ORDER_TYPES.BUY)
3193
- hasBuy = true;
3194
- if (ctx.order.type === ORDER_TYPES.SELL)
3195
- hasSell = true;
3196
- if (hasBuy && hasSell)
3197
- return true;
3198
- }
3199
- return false;
974
+ return cowRuntime.shouldExecuteCreatePairMode(this, opContexts);
3200
975
  }
3201
976
  /**
3202
977
  * Execute operations with retry on BroadcastUncertainError.
@@ -3208,36 +983,7 @@ class DEXBot {
3208
983
  * the full operations array would duplicate those creates on chain.
3209
984
  */
3210
985
  async _executeWithRetryOnUncertain(operations, opContexts) {
3211
- const MAX_RETRIES = 1;
3212
- for (let attempt = 1;; attempt++) {
3213
- try {
3214
- return await this._executeOperationsWithStrategy(operations, opContexts);
3215
- }
3216
- catch (err) {
3217
- const isRetriable = err instanceof BroadcastUncertainError
3218
- && !err.partialOnChainState
3219
- && attempt <= MAX_RETRIES;
3220
- if (isRetriable) {
3221
- this.manager.logger.log(`[COW] Broadcast uncertain (attempt ${attempt}/${MAX_RETRIES + 1}), retrying...`, 'warn');
3222
- await this._ensureCredentialDaemonWritable('COW batch retry');
3223
- // Reconcile before retry — stale ops would fail "not found".
3224
- try {
3225
- const accountRef = this.accountId || this.account?.id || this.account;
3226
- const freshChain = await chainOrders.readOpenOrders(accountRef);
3227
- if (freshChain.length > 0 && this.manager?.syncFromOpenOrders) {
3228
- await this.manager.syncFromOpenOrders(freshChain, {
3229
- skipAccounting: true,
3230
- });
3231
- }
3232
- }
3233
- catch (syncErr) {
3234
- this.manager.logger.log(`[COW] Pre-retry sync failed (non-fatal): ${syncErr?.message || syncErr}`, 'warn');
3235
- }
3236
- continue;
3237
- }
3238
- throw err;
3239
- }
3240
- }
986
+ return cowRuntime.executeWithRetryOnUncertain(this, operations, opContexts);
3241
987
  }
3242
988
  /**
3243
989
  * Execute blockchain operations with appropriate strategy (single batch or pair mode).
@@ -3246,63 +992,7 @@ class DEXBot {
3246
992
  * @returns {Promise<{result: Object, opContexts: Array}>} Execution result with contexts
3247
993
  */
3248
994
  async _executeOperationsWithStrategy(operations, opContexts) {
3249
- if (!this._shouldExecuteCreatePairMode(opContexts)) {
3250
- const result = await chainOrders.executeBatch(this.account, this.privateKey, operations);
3251
- return { result, opContexts };
3252
- }
3253
- const createEntries = [];
3254
- for (let i = 0; i < operations.length; i++) {
3255
- createEntries.push({
3256
- operation: operations[i],
3257
- context: opContexts[i],
3258
- });
3259
- }
3260
- const groups = this._buildOutsideInPairGroupsForCreateEntries(createEntries);
3261
- const mergedOperationResults = [];
3262
- const mergedRawResults = [];
3263
- const mergedContexts = [];
3264
- // Grouped execution is fail-fast, but NOT atomic across groups.
3265
- // Each group is a separate on-chain transaction; if a later group fails,
3266
- // earlier groups may already be confirmed on-chain.
3267
- for (let idx = 0; idx < groups.length; idx++) {
3268
- const group = groups[idx];
3269
- const groupOps = group.map(e => e.operation);
3270
- const groupContexts = group.map(e => e.context);
3271
- this.manager.logger.log(`[COW] Broadcasting create pair group ${idx + 1}/${groups.length} (${groupOps.length} op${groupOps.length > 1 ? 's' : ''}, outside->center)`, 'info');
3272
- let groupResult;
3273
- try {
3274
- groupResult = await chainOrders.executeBatch(this.account, this.privateKey, groupOps);
3275
- }
3276
- catch (err) {
3277
- const groupsBroadcast = idx;
3278
- const groupsTotal = groups.length;
3279
- const broadcastedOperationCount = mergedContexts.length;
3280
- this.manager.logger.log(`[COW] Grouped create execution failed at group ${idx + 1}/${groupsTotal}; ${groupsBroadcast} group(s) already broadcast (${broadcastedOperationCount} op context(s)). Partial on-chain state is possible.`, 'error');
3281
- err.partialOnChainState = groupsBroadcast > 0;
3282
- err.groupsBroadcast = groupsBroadcast;
3283
- err.groupsTotal = groupsTotal;
3284
- err.broadcastedOperationCount = broadcastedOperationCount;
3285
- throw err;
3286
- }
3287
- const groupOpResults = this._extractOperationResults(groupResult);
3288
- mergedOperationResults.push(...groupOpResults);
3289
- mergedRawResults.push(groupResult?.raw || null);
3290
- mergedContexts.push(...groupContexts);
3291
- }
3292
- return {
3293
- result: {
3294
- success: true,
3295
- raw: {
3296
- grouped: true,
3297
- groupsExecuted: groups.length,
3298
- groupResults: mergedRawResults,
3299
- },
3300
- operation_results: mergedOperationResults,
3301
- grouped: true,
3302
- groupsExecuted: groups.length
3303
- },
3304
- opContexts: mergedContexts
3305
- };
995
+ return cowRuntime.executeOperationsWithStrategy(this, operations, opContexts);
3306
996
  }
3307
997
  /**
3308
998
  * Validate that operations can be executed with available funds before broadcasting.
@@ -3314,94 +1004,7 @@ class DEXBot {
3314
1004
  * @private
3315
1005
  */
3316
1006
  _validateOperationFunds(operations, assetA, assetB) {
3317
- if (!operations || operations.length === 0) {
3318
- return { isValid: true, summary: 'No operations to validate' };
3319
- }
3320
- const { blockchainToFloat, floatToBlockchainInt, quantizeFloat } = require('./order/utils/math');
3321
- const snap = this.manager.getChainFundsSnapshot();
3322
- const netRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
3323
- const runningRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
3324
- const peakRequiredFunds = { [assetA.id]: 0, [assetB.id]: 0 };
3325
- // Sum amounts and check individual order sizes
3326
- for (const op of operations) {
3327
- if (!op?.op_data)
3328
- continue;
3329
- let sellAssetId = null;
3330
- let sellAmountInt = 0;
3331
- if (op.op_name === 'limit_order_create') {
3332
- sellAssetId = op.op_data.amount_to_sell?.asset_id;
3333
- sellAmountInt = op.op_data.amount_to_sell?.amount;
3334
- }
3335
- else if (op.op_name === 'limit_order_update') {
3336
- // In limit_order_update, new_price.base is the amount to sell
3337
- sellAssetId = op.op_data.new_price?.base?.asset_id;
3338
- sellAmountInt = op.op_data.new_price?.base?.amount;
3339
- }
3340
- if (sellAssetId && (sellAmountInt !== undefined && sellAmountInt !== null)) {
3341
- const precision = (sellAssetId === assetA.id) ? assetA.precision : assetB.precision;
3342
- const assetSymbol = (sellAssetId === assetA.id) ? assetA.symbol : assetB.symbol;
3343
- // CRITICAL SAFETY CHECK: Ensure amount is greater than zero
3344
- if (Number(sellAmountInt) <= 0) {
3345
- return {
3346
- isValid: false,
3347
- summary: `[VALIDATION] CRITICAL: Zero amount order detected for ${assetSymbol} (assetId=${sellAssetId})`,
3348
- violations: [{ asset: assetSymbol, sizeInt: sellAmountInt, reason: 'Zero amount' }]
3349
- };
3350
- }
3351
- // Track signed per-op deltas in operation order.
3352
- // CREATE consumes full amount; UPDATE consumes/releases delta.
3353
- let signedDelta = 0;
3354
- if (op.op_name === 'limit_order_update') {
3355
- const deltaAssetId = op.op_data.delta_amount_to_sell?.asset_id;
3356
- const deltaSellInt = op.op_data.delta_amount_to_sell?.amount;
3357
- if (deltaAssetId === sellAssetId && Number.isFinite(Number(deltaSellInt))) {
3358
- signedDelta = blockchainToFloat(deltaSellInt, precision);
3359
- }
3360
- }
3361
- else {
3362
- signedDelta = blockchainToFloat(sellAmountInt, precision);
3363
- }
3364
- netRequiredFunds[sellAssetId] = quantizeFloat((netRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
3365
- runningRequiredFunds[sellAssetId] = quantizeFloat((runningRequiredFunds[sellAssetId] || 0) + signedDelta, precision);
3366
- const nextPeak = Math.max(Number(peakRequiredFunds[sellAssetId] || 0), Number(runningRequiredFunds[sellAssetId] || 0));
3367
- peakRequiredFunds[sellAssetId] = quantizeFloat(nextPeak, precision);
3368
- }
3369
- }
3370
- // Calculate available funds - CRITICAL FIX: Check against FREE balance, not free+required
3371
- // Bug: Previous logic added requiredFunds to available, making validation meaningless
3372
- // Correct logic: available = chainFree (current free balance)
3373
- // If required > available, batch will fail on execution
3374
- const availableFunds = {
3375
- [assetA.id]: quantizeFloat(snap.chainFreeSell || 0, assetA.precision),
3376
- [assetB.id]: quantizeFloat(snap.chainFreeBuy || 0, assetB.precision)
3377
- };
3378
- // Check for fund violations using quantized comparison
3379
- const fundViolations = [];
3380
- for (const assetId in peakRequiredFunds) {
3381
- const required = peakRequiredFunds[assetId];
3382
- const netRequired = netRequiredFunds[assetId] || 0;
3383
- const available = availableFunds[assetId] || 0;
3384
- // Use precision-aware comparison
3385
- const prec = (assetId === assetA.id) ? assetA.precision : assetB.precision;
3386
- if (floatToBlockchainInt(required, prec) > floatToBlockchainInt(available, prec)) {
3387
- fundViolations.push({
3388
- asset: assetId === assetA.id ? assetA.symbol : assetB.symbol,
3389
- required,
3390
- netRequired,
3391
- available,
3392
- deficit: quantizeFloat(required - available, prec)
3393
- });
3394
- }
3395
- }
3396
- if (fundViolations.length > 0) {
3397
- let summary = `[VALIDATION] Fund validation FAILED:\n`;
3398
- for (const v of fundViolations) {
3399
- summary += ` ${v.asset}: peakRequired=${Format.formatAmount8(v.required)}, netRequired=${Format.formatAmount8(v.netRequired)}, available=${Format.formatAmount8(v.available)}, deficit=${Format.formatAmount8(v.deficit)}\n`;
3400
- }
3401
- return { isValid: false, summary: summary.trim(), violations: fundViolations };
3402
- }
3403
- const summary = `[VALIDATION] PASSED: ${operations.length} operations`;
3404
- return { isValid: true, summary };
1007
+ return cowRuntime.validateOperationFunds(this, operations, assetA, assetB);
3405
1008
  }
3406
1009
  /**
3407
1010
  * Resolve the ideal size from an order-like object with fallback.
@@ -3410,20 +1013,7 @@ class DEXBot {
3410
1013
  * @returns {number|null} Resolved size or null
3411
1014
  */
3412
1015
  _resolveIdealSizeForValidation(orderLike, fallbackSize = null) {
3413
- const candidates = [
3414
- orderLike?.idealSize,
3415
- orderLike?.order?.idealSize,
3416
- orderLike?.size,
3417
- orderLike?.order?.size,
3418
- fallbackSize
3419
- ];
3420
- for (const candidate of candidates) {
3421
- const numeric = Number(candidate);
3422
- if (Number.isFinite(numeric) && numeric > 0) {
3423
- return numeric;
3424
- }
3425
- }
3426
- return null;
1016
+ return cowRuntime.resolveIdealSizeForValidation(this, orderLike, fallbackSize);
3427
1017
  }
3428
1018
  /**
3429
1019
  * Validate that an order size is safe to execute (above minimum dust thresholds).
@@ -3434,7 +1024,7 @@ class DEXBot {
3434
1024
  * @returns {import('./types').OrderValidationResult}
3435
1025
  */
3436
1026
  _validateOrderSizeForExecution(size, type, orderLike = null, fallbackSize = null) {
3437
- return validateOrderSize(size, type, this.manager.assets, this.config.gridLimits?.MIN_ORDER_SIZE_FACTOR, this._resolveIdealSizeForValidation(orderLike, fallbackSize), this.config.gridLimits?.PARTIAL_DUST_THRESHOLD_PERCENTAGE);
1027
+ return cowRuntime.validateOrderSizeForExecution(this, size, type, orderLike, fallbackSize);
3438
1028
  }
3439
1029
  /**
3440
1030
  * Execute a batch of order operations if the rebalance result has executable actions.
@@ -3774,77 +1364,7 @@ class DEXBot {
3774
1364
  * @returns {Array<{type: string, id: string, order?: Object, orderId?: string, newSize?: number, newPrice?: number, newGridId?: string}>}
3775
1365
  */
3776
1366
  _buildActionsFromPlan(plan) {
3777
- const normalizedPlan = Array.isArray(plan)
3778
- ? { ordersToPlace: plan }
3779
- : (plan || {});
3780
- const { ordersToPlace = [], ordersToRotate = [], ordersToUpdate = [], ordersToCancel = [] } = normalizedPlan;
3781
- const actions = [];
3782
- for (const o of ordersToCancel) {
3783
- if (o?.orderId) {
3784
- actions.push({ type: COW_ACTIONS.CANCEL, id: o.id, orderId: o.orderId });
3785
- }
3786
- }
3787
- for (const r of ordersToRotate) {
3788
- const oldOrder = r?.oldOrder || r;
3789
- const id = oldOrder?.id || r?.id;
3790
- const orderId = oldOrder?.orderId || r?.orderId;
3791
- const newGridId = r?.newGridId || id;
3792
- const newSize = Number.isFinite(Number(r?.newSize))
3793
- ? Number(r.newSize)
3794
- : Number(r?.size || oldOrder?.size || 0);
3795
- const newPrice = Number.isFinite(Number(r?.newPrice))
3796
- ? Number(r.newPrice)
3797
- : Number(r?.price || oldOrder?.price);
3798
- const orderType = r?.type || oldOrder?.type;
3799
- if (!id || !orderId || !newGridId || !orderType || !Number.isFinite(newPrice) || !(newSize > 0))
3800
- continue;
3801
- actions.push({
3802
- type: COW_ACTIONS.UPDATE,
3803
- id,
3804
- orderId,
3805
- newGridId,
3806
- newSize,
3807
- newPrice,
3808
- order: {
3809
- id: newGridId,
3810
- type: orderType,
3811
- price: newPrice,
3812
- size: newSize
3813
- }
3814
- });
3815
- }
3816
- for (const o of ordersToUpdate) {
3817
- const partialOrder = o?.partialOrder || o;
3818
- const id = o?.id || partialOrder?.id;
3819
- const orderId = o?.orderId || partialOrder?.orderId;
3820
- const orderType = o?.type || partialOrder?.type;
3821
- const newSize = Number.isFinite(Number(o?.newSize))
3822
- ? Number(o.newSize)
3823
- : Number(partialOrder?.size || 0);
3824
- if (!id || !orderId)
3825
- continue;
3826
- actions.push({
3827
- type: COW_ACTIONS.UPDATE,
3828
- id,
3829
- orderId,
3830
- newSize,
3831
- order: {
3832
- ...(partialOrder || {}),
3833
- id,
3834
- orderId,
3835
- type: orderType,
3836
- size: newSize
3837
- }
3838
- });
3839
- }
3840
- // Run CREATE actions after UPDATE actions so same-batch downsizes can
3841
- // release balance before placements consume it.
3842
- for (const o of ordersToPlace) {
3843
- if (!o?.id)
3844
- continue;
3845
- actions.push({ type: COW_ACTIONS.CREATE, id: o.id, order: o });
3846
- }
3847
- return actions;
1367
+ return cowRuntime.buildActionsFromPlan(this, plan);
3848
1368
  }
3849
1369
  /**
3850
1370
  * Build a COW result object (workingGrid + actions) from a simple plan.
@@ -3852,78 +1372,7 @@ class DEXBot {
3852
1372
  * @returns {{workingGrid: import('./types').WorkingGrid, workingIndexes: Object, workingBoundary: number, actions: Array}}
3853
1373
  */
3854
1374
  _buildCowResultFromPlan(plan) {
3855
- const { WorkingGrid } = require('./order/working_grid');
3856
- const workingGrid = new WorkingGrid(this.manager.orders, {
3857
- baseVersion: Number.isFinite(Number(this.manager._gridVersion)) ? this.manager._gridVersion : 0
3858
- });
3859
- const workingBoundary = this.manager.boundaryIdx;
3860
- const actions = this._buildActionsFromPlan(plan);
3861
- // Project planned actions into working grid so COW commit carries intended transitions.
3862
- for (const action of actions) {
3863
- if (action.type === COW_ACTIONS.CANCEL) {
3864
- const current = workingGrid.get(action.id);
3865
- if (!current)
3866
- continue;
3867
- workingGrid.set(action.id, convertToSpreadPlaceholder(current));
3868
- }
3869
- else if (action.type === COW_ACTIONS.CREATE) {
3870
- if (!action.id || !action.order)
3871
- continue;
3872
- const current = workingGrid.get(action.id) || { id: action.id };
3873
- workingGrid.set(action.id, {
3874
- ...current,
3875
- ...action.order,
3876
- id: action.id,
3877
- state: ORDER_STATES.VIRTUAL,
3878
- orderId: null
3879
- });
3880
- }
3881
- else if (action.type === COW_ACTIONS.UPDATE) {
3882
- if (action.newGridId && action.newGridId !== action.id) {
3883
- const current = workingGrid.get(action.id);
3884
- if (current) {
3885
- workingGrid.set(action.id, convertToSpreadPlaceholder(current));
3886
- }
3887
- const targetId = action.newGridId;
3888
- const targetCurrent = workingGrid.get(targetId) || { id: targetId };
3889
- const rotatedSize = Number.isFinite(Number(action.newSize))
3890
- ? Number(action.newSize)
3891
- : Number(targetCurrent.size || 0);
3892
- const rotatedPrice = Number.isFinite(Number(action.newPrice))
3893
- ? Number(action.newPrice)
3894
- : Number(action.order?.price ?? targetCurrent.price);
3895
- workingGrid.set(targetId, {
3896
- ...targetCurrent,
3897
- ...(action.order || {}),
3898
- id: targetId,
3899
- size: rotatedSize,
3900
- price: rotatedPrice,
3901
- state: ORDER_STATES.VIRTUAL,
3902
- orderId: null
3903
- });
3904
- continue;
3905
- }
3906
- const current = workingGrid.get(action.id);
3907
- if (!current)
3908
- continue;
3909
- const newSize = Number.isFinite(Number(action.newSize))
3910
- ? Number(action.newSize)
3911
- : Number(current.size || 0);
3912
- workingGrid.set(action.id, {
3913
- ...current,
3914
- ...(action.order || {}),
3915
- id: action.id,
3916
- orderId: action.orderId || current.orderId,
3917
- size: newSize
3918
- });
3919
- }
3920
- }
3921
- return {
3922
- workingGrid,
3923
- workingIndexes: workingGrid.getIndexes(),
3924
- workingBoundary,
3925
- actions
3926
- };
1375
+ return cowRuntime.buildCowResultFromPlan(this, plan);
3927
1376
  }
3928
1377
  /**
3929
1378
  * Restore skipped update slots in the working grid to master state.
@@ -3933,16 +1382,7 @@ class DEXBot {
3933
1382
  * @returns {void}
3934
1383
  */
3935
1384
  _restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedSlotIds, skippedCount = 0) {
3936
- if (!workingGrid || !skippedSlotIds || skippedSlotIds.size === 0) {
3937
- return;
3938
- }
3939
- const masterVersion = Number.isFinite(Number(this.manager?._gridVersion))
3940
- ? Number(this.manager._gridVersion)
3941
- : undefined;
3942
- for (const slotId of skippedSlotIds) {
3943
- workingGrid.syncFromMaster(this.manager.orders, slotId, masterVersion);
3944
- }
3945
- this.manager.logger.log(`[COW] Restored ${skippedSlotIds.size} slot(s) after ${skippedCount} skipped update action(s).`, 'debug');
1385
+ return cowRuntime.restoreSkippedUpdateSlotsInWorkingGrid(this, workingGrid, skippedSlotIds, skippedCount);
3946
1386
  }
3947
1387
  /**
3948
1388
  * COW broadcast: Execute blockchain operations and commit working grid on success.
@@ -3952,751 +1392,10 @@ class DEXBot {
3952
1392
  * @private
3953
1393
  */
3954
1394
  async _updateOrdersOnChainBatchCOW(cowResult) {
3955
- this._currentCycleId = (Number.isFinite(Number(this._currentCycleId)) ? Number(this._currentCycleId) : 0) + 1;
3956
- const { workingGrid, workingIndexes, workingBoundary, actions } = cowResult;
3957
- if (this.config.dryRun) {
3958
- const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
3959
- const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
3960
- const updateCount = actions.filter(a => a.type === COW_ACTIONS.UPDATE).length;
3961
- if (cancelCount > 0)
3962
- this.manager.logger.log(`Dry run: would cancel ${cancelCount} orders`, 'info');
3963
- if (createCount > 0)
3964
- this.manager.logger.log(`Dry run: would place ${createCount} new orders`, 'info');
3965
- if (updateCount > 0)
3966
- this.manager.logger.log(`Dry run: would update ${updateCount} orders`, 'info');
3967
- return { executed: true, hadRotation: false };
3968
- }
3969
- const createSlotValidation = validateCreateTargetSlots(actions, this.manager?.orders);
3970
- if (!createSlotValidation.isValid) {
3971
- for (const violation of createSlotValidation.violations) {
3972
- this.manager.logger.log(`[COW] Rejecting CREATE for occupied slot ${violation.targetId}: ` +
3973
- `existing orderId=${violation.currentOrderId}, type=${violation.currentType}, state=${violation.currentState}`, 'error');
3974
- }
3975
- return {
3976
- executed: false,
3977
- aborted: true,
3978
- reason: 'CREATE_SLOT_OCCUPIED',
3979
- violations: createSlotValidation.violations,
3980
- hadRotation: false
3981
- };
3982
- }
3983
- const hasCreateActions = actions.some(action => action.type === COW_ACTIONS.CREATE);
3984
- // Gap 5: Recovery exhausted — block all CREATES. The bot has exhausted
3985
- // its recovery attempt budget and needs a fill or sync cycle to reset.
3986
- // Existing orders remain active and are monitored, but no new orders
3987
- // are placed until recovery resets.
3988
- if (hasCreateActions && this.manager?._recoveryExhaustedAt) {
3989
- const exhaustedAge = Date.now() - this.manager._recoveryExhaustedAt;
3990
- this.manager.logger.log?.(`[RECOVERY-EXHAUSTED] Blocking ${actions.filter(a => a.type === COW_ACTIONS.CREATE).length} CREATE(s) ` +
3991
- `(exhausted ${(exhaustedAge / 1000).toFixed(0)}s ago). ` +
3992
- `Waiting for next fill or sync cycle to reset recovery state.`, 'warn');
3993
- return {
3994
- executed: false,
3995
- aborted: true,
3996
- reason: 'RECOVERY_EXHAUSTED',
3997
- hadRotation: false
3998
- };
3999
- }
4000
- const unmatchedChainOrders = Array.isArray(this.manager?._lastUnmatchedChainOrders)
4001
- ? this.manager._lastUnmatchedChainOrders
4002
- : [];
4003
- // PENDING_BROADCASTS: an earlier batch timed out at the credential
4004
- // daemon and the chain status of its CREATEs is still unknown. The
4005
- // recovery path (_reconcileAfterUncertainBroadcast) is responsible
4006
- // for resolving it. We refuse to publish a fresh CREATE batch until
4007
- // recovery runs, otherwise we risk double-publishing or stacking
4008
- // orphan orders on top of potentially-orphaned ones.
4009
- const pendingBroadcasts = (this.manager && this.manager._pendingBroadcasts instanceof Map)
4010
- ? Array.from(this.manager._pendingBroadcasts.values())
4011
- : [];
4012
- if (hasCreateActions && (unmatchedChainOrders.length > 0 || pendingBroadcasts.length > 0)) {
4013
- // ---- Handle pending broadcasts first ----
4014
- if (pendingBroadcasts.length > 0) {
4015
- this.manager.logger.log(`[COW] Rejecting CREATE batch: ${pendingBroadcasts.length} pending broadcast(s) from a prior uncertain ` +
4016
- `broadcast. Running recovery before placing replacement orders.`, 'error');
4017
- if (typeof this.manager.requestStructuralGridResync === 'function') {
4018
- if (this.manager._recoveryState)
4019
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
4020
- await this.manager.requestStructuralGridResync('pending broadcasts before COW create', { pendingBroadcasts: pendingBroadcasts.map(p => p.slotId) });
4021
- }
4022
- try {
4023
- await this._reconcileAfterUncertainBroadcast(new BroadcastUncertainError('rejected CREATE batch had pending broadcasts', {
4024
- operations: pendingBroadcasts.map(p => p.order),
4025
- accountName: this.account,
4026
- batchId: this._currentBatchId || null,
4027
- payload: null,
4028
- timeoutMs: null
4029
- }), []);
4030
- }
4031
- catch (recoverErr) {
4032
- this.manager.logger.log(`[COW] Recovery from pending broadcasts failed: ${recoverErr?.message || recoverErr}`, 'error');
4033
- }
4034
- return {
4035
- executed: false,
4036
- aborted: true,
4037
- reason: 'PENDING_BROADCASTS',
4038
- hadRotation: false
4039
- };
4040
- }
4041
- // ---- Unmatched orders: adopt instead of cancel ----
4042
- // Unmatched orders are legitimate on-chain positions the grid
4043
- // hasn't adopted yet. Cancelling them destroys capital and
4044
- // creates gaps in the order book. Instead, re-sync to adopt
4045
- // them into the grid, then reject (the sync invalidated the
4046
- // working grid, so the existing COW plan is stale).
4047
- // NOTE: syncFromOpenOrders may be a partial no-op under certain
4048
- // lock conditions (e.g. _fillProcessingLock + !isReentrant).
4049
- // Structural resync scheduled below handles adoption regardless.
4050
- const unmatchedSample = unmatchedChainOrders
4051
- .slice(0, 3)
4052
- .map(o => this._formatUnmatchedChainOrderForLog(o))
4053
- .join(' | ');
4054
- this.manager.logger.log(`[COW] ${unmatchedChainOrders.length} unmatched chain order(s) blocking CREATES ` +
4055
- (unmatchedSample ? `(${unmatchedSample})` : '') +
4056
- ` — adopting via sync instead of cancelling`, 'info');
4057
- try {
4058
- const accountRef = this.account;
4059
- const freshSnapshot = await chainOrders.readOpenOrders(accountRef);
4060
- if (freshSnapshot && freshSnapshot.length > 0) {
4061
- const syncResult = await this.manager.syncFromOpenOrders(freshSnapshot, {
4062
- skipAccounting: true,
4063
- });
4064
- // Only overwrite _lastUnmatchedChainOrders when sync actually
4065
- // processed orders (filled + updated + corrected > 0).
4066
- // Force-release and lock-contention early-exit paths return
4067
- // empty arrays without touching _lastUnmatchedChainOrders —
4068
- // overwriting with [] would drop stale unmatched entries.
4069
- if (syncResult && Array.isArray(syncResult.unmatchedChainOrders)) {
4070
- const processed = (syncResult.filledOrders?.length || 0) +
4071
- (syncResult.updatedOrders?.length || 0) +
4072
- (syncResult.ordersNeedingCorrection?.length || 0);
4073
- if (processed > 0) {
4074
- this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders;
4075
- this.manager.logger.log(`[COW] Adopted chain order(s) via sync: ${processed} processed, ` +
4076
- `${syncResult.unmatchedChainOrders.length} still unmatched`, 'info');
4077
- }
4078
- else {
4079
- // processed === 0 with unmatched still present means the
4080
- // sync could not adopt the unmatched chain orders. This is
4081
- // normal when called re-entrantly (inside _fillProcessingLock):
4082
- // the sync engine runs inline and either timed out (unlikely
4083
- // for a re-entrant call) or all chain orders were already
4084
- // matched — leaving only stale unmatched entries.
4085
- const syncUnmatchedCount = syncResult.unmatchedChainOrders.length;
4086
- this.manager.logger.log(`[COW] Sync returned without processing (processed=0, ` +
4087
- `unmatched=${syncUnmatchedCount} in result, ` +
4088
- `_lastUnmatchedChainOrders=${unmatchedChainOrders.length}). ` +
4089
- `Structural resync will handle adoption.`, syncUnmatchedCount > 0 ? 'warn' : 'debug');
4090
- // If the sync result itself has unmatched entries that
4091
- // differ from _lastUnmatchedChainOrders, adopt them now
4092
- // so the stale tracker is more accurate for the resync.
4093
- if (syncUnmatchedCount > 0 && syncUnmatchedCount !== unmatchedChainOrders.length) {
4094
- this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders.map((o) => ({ ...o }));
4095
- this.manager.logger.log(`[COW] Updated _lastUnmatchedChainOrders from sync result: ` +
4096
- `${unmatchedChainOrders.length} → ${syncUnmatchedCount}`, 'debug');
4097
- }
4098
- }
4099
- }
4100
- }
4101
- }
4102
- catch (syncErr) {
4103
- this.manager.logger.log(`[COW] Failed to sync/unmatched orders: ${syncErr?.message || syncErr}`, 'warn');
4104
- }
4105
- // Working grid is stale after master grid mutation from sync.
4106
- // Request structural resync to rebuild the grid on the next cycle.
4107
- if (typeof this.manager.requestStructuralGridResync === 'function') {
4108
- if (this.manager._recoveryState)
4109
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
4110
- await this.manager.requestStructuralGridResync('unmatched chain orders before COW create', { unmatchedChainOrders: unmatchedChainOrders });
4111
- }
4112
- this.manager.logger.log(`[COW] Rejecting CREATE batch after sync: working grid invalidated by master mutation`, 'info');
4113
- return {
4114
- executed: false,
4115
- aborted: true,
4116
- reason: 'UNMATCHED_CHAIN_ORDERS',
4117
- hadRotation: false
4118
- };
4119
- }
4120
- const { assetA, assetB } = this.manager.assets;
4121
- const operations = [];
4122
- const opContexts = [];
4123
- const skippedUpdateSlotIds = new Set();
4124
- let skippedUpdateCount = 0;
4125
- // Collect IDs to lock from actions
4126
- const idsToLock = new Set();
4127
- for (const action of actions) {
4128
- if (action.type === COW_ACTIONS.CANCEL && action.orderId) {
4129
- idsToLock.add(action.orderId);
4130
- if (action.id)
4131
- idsToLock.add(action.id);
4132
- }
4133
- else if (action.type === COW_ACTIONS.CREATE && action.id) {
4134
- idsToLock.add(action.id);
4135
- }
4136
- else if (action.type === COW_ACTIONS.UPDATE && action.orderId) {
4137
- idsToLock.add(action.orderId);
4138
- if (action.id)
4139
- idsToLock.add(action.id);
4140
- }
4141
- }
4142
- // Apply shadow locks
4143
- this.manager.lockOrders(idsToLock);
4144
- try {
4145
- this._batchInFlight = true;
4146
- this._markGridActivity('batch start');
4147
- this.manager._setRebalanceState(REBALANCE_STATES.BROADCASTING);
4148
- this.manager.startBroadcasting();
4149
- // Build operations from actions
4150
- for (const action of actions) {
4151
- if (action.type === COW_ACTIONS.CANCEL) {
4152
- try {
4153
- const op = await chainOrders.buildCancelOrderOp(this.account, action.orderId);
4154
- operations.push(op);
4155
- const order = this.manager.orders.get(action.id) || { id: action.id, orderId: action.orderId };
4156
- opContexts.push({ kind: 'cancel', order });
4157
- }
4158
- catch (err) {
4159
- const orderNotFound = /\bnot found\b/i.test(err.message) || /\bdoes not exist\b/i.test(err.message);
4160
- if (orderNotFound) {
4161
- this.manager.logger.log(`[COW] Cancel skipped for ${action.id} (${action.orderId}): order already removed from chain`, 'debug');
4162
- }
4163
- else {
4164
- this.manager.logger.log(`Failed to prepare cancel op for ${action.id}: ${err.message}`, 'error');
4165
- }
4166
- }
4167
- }
4168
- else if (action.type === COW_ACTIONS.CREATE) {
4169
- try {
4170
- const order = action.order;
4171
- const sizeValidation = this._validateOrderSizeForExecution(order.size, order.type, order, order.size);
4172
- if (!sizeValidation.isValid) {
4173
- this.manager.logger.log(`Skipping create op for ${action.id}: ${sizeValidation.reason}`, 'warn');
4174
- continue;
4175
- }
4176
- const liveSlot = this.manager.orders.get(order.id);
4177
- const plannedPrice = Number(order.price);
4178
- const livePrice = liveSlot ? Number(liveSlot.price) : NaN;
4179
- const priceDrift = Number.isFinite(plannedPrice) && Number.isFinite(livePrice)
4180
- ? Math.abs(livePrice - plannedPrice)
4181
- : 0;
4182
- const effectiveOrder = (priceDrift > 0)
4183
- ? { ...order, price: livePrice, size: order.size, type: order.type }
4184
- : order;
4185
- if (priceDrift > 0) {
4186
- this.manager.logger.log(`[COW] Pre-broadcast price freshness: slot ${order.id} ` +
4187
- `drifted from planned=${plannedPrice} to live=${livePrice} ` +
4188
- `(diff=${priceDrift}); rebuilding CREATE op with live price.`, 'debug');
4189
- }
4190
- const args = buildCreateOrderArgs(effectiveOrder, assetA, assetB);
4191
- const buildResult = await chainOrders.buildCreateOrderOp(this.account, args.amountToSell, args.sellAssetId, args.minToReceive, args.receiveAssetId, null);
4192
- if (!buildResult) {
4193
- this.manager.logger.log(`Skipping create op for ${action.id}: amounts would round to 0 on blockchain`, 'warn');
4194
- continue;
4195
- }
4196
- operations.push(buildResult.op);
4197
- opContexts.push({ kind: 'create', id: order.id, order: effectiveOrder, args, finalInts: buildResult.finalInts });
4198
- this._recordPendingBroadcast({
4199
- opIndex: operations.length - 1,
4200
- ctxIndex: opContexts.length - 1,
4201
- order: effectiveOrder,
4202
- finalInts: buildResult.finalInts
4203
- });
4204
- }
4205
- catch (err) {
4206
- this.manager.logger.log(`Failed to prepare create op for ${action.id}: ${err.message}`, 'error');
4207
- }
4208
- }
4209
- else if (action.type === COW_ACTIONS.UPDATE) {
4210
- try {
4211
- // Rotation update: move existing on-chain order to a new slot/price.
4212
- if (action.newGridId && action.newGridId !== action.id) {
4213
- const masterOrder = this.manager.orders.get(action.id);
4214
- const orderType = action.order?.type || masterOrder?.type;
4215
- const newPrice = Number.isFinite(Number(action.newPrice))
4216
- ? Number(action.newPrice)
4217
- : Number(action.order?.price);
4218
- const newSize = Number.isFinite(Number(action.newSize))
4219
- ? Number(action.newSize)
4220
- : Number(action.order?.size || 0);
4221
- if (!masterOrder || !action.orderId || !orderType || !Number.isFinite(newPrice) || newSize <= 0) {
4222
- continue;
4223
- }
4224
- const rotationSizeValidation = this._validateOrderSizeForExecution(newSize, orderType, action.order, newSize);
4225
- if (!rotationSizeValidation.isValid) {
4226
- this.manager.logger.log(`Skipping rotation update ${action.id} -> ${action.newGridId}: ${rotationSizeValidation.reason}`, 'warn');
4227
- continue;
4228
- }
4229
- const { amountToSell, minToReceive } = buildCreateOrderArgs({ type: orderType, size: newSize, price: newPrice }, assetA, assetB);
4230
- const buildResult = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell, minToReceive, newPrice, orderType }, masterOrder.rawOnChain || null);
4231
- if (!buildResult) {
4232
- skippedUpdateCount++;
4233
- if (action.id)
4234
- skippedUpdateSlotIds.add(action.id);
4235
- if (action.newGridId)
4236
- skippedUpdateSlotIds.add(action.newGridId);
4237
- this.manager.logger.log(`[COW] Skipping rotation update ${action.id} -> ${action.newGridId}: no blockchain delta`, 'debug');
4238
- continue;
4239
- }
4240
- operations.push(buildResult.op);
4241
- opContexts.push({
4242
- kind: 'rotation',
4243
- rotation: {
4244
- oldOrder: { ...masterOrder },
4245
- newGridId: action.newGridId,
4246
- newPrice,
4247
- newSize,
4248
- type: orderType
4249
- },
4250
- finalInts: buildResult.finalInts
4251
- });
4252
- continue;
4253
- }
4254
- const newSize = Number.isFinite(Number(action.newSize))
4255
- ? Number(action.newSize)
4256
- : Number(action.order?.size || 0);
4257
- const masterOrder = this.manager.orders.get(action.id);
4258
- const orderType = action.order?.type || masterOrder?.type;
4259
- const cachedRawOnChain = masterOrder?.rawOnChain || action.order?.rawOnChain || null;
4260
- const op = await chainOrders.buildUpdateOrderOp(this.account, action.orderId, { amountToSell: newSize, orderType }, cachedRawOnChain);
4261
- if (!op) {
4262
- skippedUpdateCount++;
4263
- if (action.id)
4264
- skippedUpdateSlotIds.add(action.id);
4265
- if (action.newGridId)
4266
- skippedUpdateSlotIds.add(action.newGridId);
4267
- this.manager.logger.log(`[COW] Skipping size update ${action.id} (${action.orderId}): no blockchain delta`, 'debug');
4268
- continue;
4269
- }
4270
- operations.push(op.op);
4271
- const partialOrder = masterOrder || {
4272
- id: action.id,
4273
- orderId: action.orderId,
4274
- type: orderType
4275
- };
4276
- opContexts.push({ kind: 'size-update', updateInfo: { partialOrder, newSize }, finalInts: op.finalInts });
4277
- // Catch for both rotation-update and size-update branches.
4278
- }
4279
- catch (err) {
4280
- const orderNotFound = /\bnot found\b/i.test(err.message) || /\bdoes not exist\b/i.test(err.message);
4281
- if (orderNotFound) {
4282
- try {
4283
- const fbOrder = action.order || this.manager.orders.get(action.id);
4284
- const fbType = fbOrder?.type;
4285
- const fbSize = action.newSize || fbOrder?.size || 0;
4286
- // Live price from TARGET slot (same pattern as CREATE path).
4287
- const targetSlotId = action.newGridId || action.id;
4288
- const plannedPrice = action.newPrice || action.order?.price || 0;
4289
- const liveSlotForPrice = this.manager.orders.get(targetSlotId);
4290
- const livePrice = liveSlotForPrice ? Number(liveSlotForPrice.price) : NaN;
4291
- const priceDrift = Number.isFinite(plannedPrice) && Number.isFinite(livePrice)
4292
- ? Math.abs(livePrice - plannedPrice)
4293
- : 0;
4294
- const fbPrice = (priceDrift > 0) ? livePrice : plannedPrice;
4295
- if (priceDrift > 0) {
4296
- this.manager.logger.log(`[COW] CREATE fallback price drift for ${action.id} -> ${targetSlotId}: ` +
4297
- `planned=${plannedPrice} live=${livePrice} (diff=${priceDrift})`, 'debug');
4298
- }
4299
- // Same size gate as regular CREATE path.
4300
- const sizeCheck = this._validateOrderSizeForExecution(fbSize, fbType, fbOrder, fbSize);
4301
- if (!sizeCheck.isValid) {
4302
- this.manager.logger.log(`[COW] CREATE fallback for ${action.id} rejected by size validation: ${sizeCheck.reason}`, 'warn');
4303
- }
4304
- else if (fbType && fbSize > 0 && fbPrice > 0) {
4305
- const fbArgs = buildCreateOrderArgs({ type: fbType, size: fbSize, price: fbPrice }, assetA, assetB);
4306
- const fbResult = await chainOrders.buildCreateOrderOp(this.account, fbArgs.amountToSell, fbArgs.sellAssetId, fbArgs.minToReceive, fbArgs.receiveAssetId, null);
4307
- if (fbResult) {
4308
- operations.push(fbResult.op);
4309
- opContexts.push({
4310
- kind: 'create',
4311
- id: targetSlotId,
4312
- order: { id: targetSlotId, type: fbType, price: fbPrice, size: fbSize },
4313
- args: { amountToSell: fbArgs.amountToSell, minToReceive: fbArgs.minToReceive },
4314
- finalInts: fbResult.finalInts
4315
- });
4316
- this._recordPendingBroadcast({
4317
- opIndex: operations.length - 1,
4318
- ctxIndex: opContexts.length - 1,
4319
- order: { id: targetSlotId, type: fbType, price: fbPrice, size: fbSize },
4320
- finalInts: fbResult.finalInts
4321
- });
4322
- this.manager.logger.log(`[COW] Recovered "not found" for ${action.id}: converted UPDATE to CREATE for slot ${targetSlotId}`, 'warn');
4323
- continue;
4324
- }
4325
- }
4326
- }
4327
- catch (fbErr) {
4328
- this.manager.logger.log(`[COW] CREATE fallback also failed for ${action.id}: ${fbErr.message}`, 'warn');
4329
- }
4330
- }
4331
- this.manager.logger.log(`Failed to prepare update op for ${action.id}: ${err.message}`, 'error');
4332
- }
4333
- }
4334
- }
4335
- if (skippedUpdateCount > 0) {
4336
- this._restoreSkippedUpdateSlotsInWorkingGrid(workingGrid, skippedUpdateSlotIds, skippedUpdateCount);
4337
- }
4338
- if (operations.length === 0) {
4339
- this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
4340
- return { executed: false, hadRotation: false };
4341
- }
4342
- // Validate funds before broadcasting
4343
- const validation = this._validateOperationFunds(operations, assetA, assetB);
4344
- this.manager.logger.log(validation.summary, validation.isValid ? 'info' : 'warn');
4345
- if (!validation.isValid) {
4346
- this.manager.logger.log(`Skipping batch broadcast: ${validation.violations.length} fund violation(s) detected`, 'warn');
4347
- this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
4348
- return { executed: false, hadRotation: false };
4349
- }
4350
- await this._ensureCredentialDaemonWritable('COW batch broadcast');
4351
- // Execute batch
4352
- this.manager.logger.log(`[COW] Broadcasting batch with ${operations.length} operations...`, 'info');
4353
- this._lastBroadcastHeartbeatAt = Date.now();
4354
- const execution = await this._executeWithRetryOnUncertain(operations, opContexts);
4355
- const result = execution.result;
4356
- const executedContexts = execution.opContexts;
4357
- // Process results and commit on success
4358
- this.manager.pauseFundRecalc();
4359
- try {
4360
- this.manager._throwOnIllegalState = true;
4361
- if (result.success) {
4362
- // Pre-commit integrity: only CREATE ops require returned chainOrderIds.
4363
- // Cancel/update operation results may be empty depending on the broadcaster.
4364
- const preCommitResults = this._extractOperationResults(result, 'pre-commit-integrity');
4365
- const missingCreateResults = this._findMissingCreateResultContexts(preCommitResults, executedContexts);
4366
- if (missingCreateResults.length > 0) {
4367
- const missingSlots = missingCreateResults
4368
- .map(item => item.ctx?.order?.id || item.ctx?.id || `op-${item.index}`)
4369
- .join(', ');
4370
- this.manager.logger.log(`[COW] Refusing to commit working grid: ${missingCreateResults.length} CREATE op(s) ` +
4371
- `returned no chainOrderId (${missingSlots}). Discarding working grid and syncing from chain.`, 'error');
4372
- this.manager._clearWorkingGridRef();
4373
- this.manager._setRebalanceState(REBALANCE_STATES.NORMAL);
4374
- this._markMissingCreateResultsAsStructuralBlocker(missingCreateResults);
4375
- await this._recoverAfterMissingCreateResults('missing create operation results');
4376
- return {
4377
- executed: false,
4378
- hadRotation: false,
4379
- missingCreateResults: missingCreateResults.map(item => ({
4380
- index: item.index,
4381
- slotId: item.ctx?.order?.id || item.ctx?.id || null
4382
- }))
4383
- };
4384
- }
4385
- // SUCCESS: Commit working grid to master (atomic swap)
4386
- this.manager.logger.log('[COW] Blockchain success - committing working grid to master', 'info');
4387
- // RC-FIX: skipRecalc prevents invariant violation before optimistic accounting
4388
- await this.manager._commitWorkingGrid(workingGrid, workingIndexes, workingBoundary, { skipRecalc: true });
4389
- // Commitment accounting is handled in real-time by
4390
- // updateOptimisticFreeBalance when capital is committed to orders.
4391
- // The old post-batch deduction path was removed to avoid double-counting.
4392
- // Process batch results for logging/metrics
4393
- const batchResult = await this._processBatchResults(result, executedContexts);
4394
- // Persist to disk. CRITICAL: working grid was already committed
4395
- // to master above; if persistence is skipped or validation fails
4396
- // here, the on-disk snapshot will be older than the in-memory
4397
- // master. Retry once and surface the failure explicitly so the
4398
- // next cycle / shutdown can recover.
4399
- const persistResult = await this.manager.persistGrid();
4400
- if (persistResult && (persistResult.skipped || persistResult.isValid === false)) {
4401
- this.manager.logger.log(`[COW][PERSIST-GUARD] First persist attempt was ` +
4402
- `${persistResult.skipped ? 'skipped' : 'invalid'} ` +
4403
- `(${persistResult.reason || 'no reason'}); retrying once before ` +
4404
- `clearing working grid reference.`, 'warn');
4405
- this.manager._persistenceWarning = persistResult;
4406
- const retryResult = await this.manager.persistGrid();
4407
- if (retryResult && (retryResult.skipped || retryResult.isValid === false)) {
4408
- this.manager.logger.log(`[COW][PERSIST-GUARD] Retry also skipped/invalid ` +
4409
- `(${retryResult.reason || 'no reason'}). Master grid in memory ` +
4410
- `is ahead of disk snapshot; structural resync requested.`, 'error');
4411
- if (typeof this.manager.requestStructuralGridResync === 'function') {
4412
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: true };
4413
- await this.manager.requestStructuralGridResync('persistence guard triggered after COW batch', { persistReason: retryResult.reason || 'unknown' });
4414
- }
4415
- }
4416
- else {
4417
- delete this.manager._persistenceWarning;
4418
- }
4419
- }
4420
- else if (this.manager._persistenceWarning) {
4421
- delete this.manager._persistenceWarning;
4422
- }
4423
- this._metrics.batchesExecuted++;
4424
- this.manager._clearWorkingGridRef();
4425
- this._clearPendingBroadcasts();
4426
- return { executed: true, hadRotation: true, ...batchResult };
4427
- }
4428
- else {
4429
- // FAILURE: Working grid discarded, master unchanged
4430
- this.manager.logger.log('[COW] Blockchain failed - working grid discarded, master unchanged', 'warn');
4431
- this.manager._clearWorkingGridRef();
4432
- this._clearPendingBroadcasts();
4433
- return { executed: false, hadRotation: false, ...result };
4434
- }
4435
- }
4436
- finally {
4437
- this.manager._throwOnIllegalState = false;
4438
- // Keep broadcasting true during resumeFundRecalc to skip invariant checks
4439
- // that would fail due to stale accountTotals (not yet refreshed from blockchain)
4440
- await this.manager.resumeFundRecalc();
4441
- this.manager.stopBroadcasting();
4442
- const createCount = actions.filter(a => a.type === COW_ACTIONS.CREATE).length;
4443
- const cancelCount = actions.filter(a => a.type === COW_ACTIONS.CANCEL).length;
4444
- this.manager.logger.logFundsStatus(this.manager, `AFTER COW batch (created=${createCount}, cancelled=${cancelCount})`);
4445
- }
4446
- }
4447
- catch (err) {
4448
- this.manager.logger.log(`[COW] Batch transaction failed: ${err.message}`, 'error');
4449
- if (err?.partialOnChainState) {
4450
- this.manager.logger.log(`[COW] Non-atomic grouped execution detected (${err.groupsBroadcast}/${err.groupsTotal} groups broadcast). Local rollback cannot undo confirmed on-chain operations; next sync/reconcile will converge state.`, 'warn');
4451
- }
4452
- this.manager.stopBroadcasting();
4453
- this.manager._clearWorkingGridRef();
4454
- // BROADCAST_UNCERTAIN: the credential daemon timed out (or hit its
4455
- // inner deadline) and the chain status of the planned CREATEs is
4456
- // unknown. Run the reconcile-then-decide recovery path: read the
4457
- // chain, match each pending broadcast by fingerprint, adopt any
4458
- // chain-side matches, and discard the rest. We MUST NOT throw —
4459
- // throwing would re-enter the catch loop on the next attempt and
4460
- // potentially double-publish.
4461
- if (err instanceof BroadcastUncertainError) {
4462
- return await this._reconcileAfterUncertainBroadcast(err, opContexts);
4463
- }
4464
- // Handle hard abort
4465
- const hardAbortResult = await this._handleBatchHardAbort(err, 'COW batch processing', operations.length);
4466
- if (hardAbortResult)
4467
- return hardAbortResult;
4468
- // Check for stale orders — filled in the ~1.5s broadcast window after our plan was built.
4469
- const staleOrderIds = new Set();
4470
- const patterns = [
4471
- /Limit order (1\.7\.\d+) does not exist/g,
4472
- /Unable to find Object (1\.7\.\d+)/g,
4473
- /object (1\.7\.\d+) (?:does not exist|not found)/gi
4474
- ];
4475
- for (const pattern of patterns) {
4476
- let m;
4477
- while ((m = pattern.exec(err.message)) !== null) {
4478
- staleOrderIds.add(m[1]);
4479
- }
4480
- }
4481
- // "Cannot deduct all or more from order than order contains" means the order still
4482
- // exists, but its on-chain size shrank during the broadcast window. That is not an
4483
- // explicit stale/missing-order signal, so reconcile from chain instead of virtualizing.
4484
- if (/Cannot deduct all or more from order than order contains/.test(err.message)) {
4485
- return await this._recoverBatchSizeDrift(err, opContexts);
4486
- }
4487
- if (staleOrderIds.size > 0) {
4488
- // Recover explicit missing-order failures without aborting the entire fill cycle.
4489
- // Use the manager update path so indexes, working-grid sync, accounting, and
4490
- // persistence stay coherent.
4491
- return await this._recoverExplicitStaleOrders(staleOrderIds, 'cow-stale-order-cleanup');
4492
- }
4493
- throw err;
4494
- }
4495
- finally {
4496
- this._batchInFlight = false;
4497
- this._markGridActivity('batch end');
4498
- this.manager.unlockOrders(idsToLock);
4499
- if (!this._shuttingDown && this._incomingFillQueue.length > 0) {
4500
- this._scheduleFillConsumerRestart(chainOrders);
4501
- }
4502
- }
1395
+ return cowRuntime.updateOrdersOnChainBatchCOW(this, cowResult);
4503
1396
  }
4504
- /**
4505
- * Process results from batch transaction execution.
4506
- * Updates order state, synchronizes with chain, and deducts BTS fees.
4507
- * @param {Object} result - Transaction result from executeBatch
4508
- * @param {Array} opContexts - Operation context array with operation metadata (must be 1:1 with result.operation_results)
4509
- * @returns {Object} Result with { executed: boolean, hadRotation: boolean }
4510
- * @private
4511
- */
4512
1397
  async _processBatchResults(result, opContexts) {
4513
- const results = this._extractOperationResults(result, '_processBatchResults');
4514
- const { getAssetFees } = require('./order/utils/math');
4515
- // IMPORTANT: Call without amount to get fee schedule fields
4516
- // ({ createFee, updateFee, ... }), not proceeds projection fields.
4517
- const btsFeeData = getAssetFees('BTS');
4518
- let hadRotation = false;
4519
- let updateOperationCount = 0;
4520
- const updatesToApply = [];
4521
- for (let i = 0; i < opContexts.length; i++) {
4522
- const ctx = opContexts[i];
4523
- const res = results[i];
4524
- if (ctx.kind === 'cancel') {
4525
- this.manager.logger.log(`Cancelled surplus order ${ctx.order.id} (${ctx.order.orderId})`, 'info');
4526
- const oldOrder = ctx.order;
4527
- const committedOrder = oldOrder?.id ? this.manager.orders.get(oldOrder.id) : null;
4528
- // The COW commit already updated manager.orders. Apply ONLY the optimistic
4529
- // accounting transition using pre-commit -> committed order states.
4530
- if (oldOrder && committedOrder && this.manager.accountant) {
4531
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, committedOrder, 'fill-cancel', btsFeeData?.cancelFee || 0, false);
4532
- }
4533
- }
4534
- else if (ctx.kind === 'size-update') {
4535
- const oldOrder = ctx.updateInfo.partialOrder;
4536
- const ord = this.manager.orders.get(oldOrder.id);
4537
- // Apply optimistic accounting from pre-commit -> committed state,
4538
- // including blockchain update fee deduction.
4539
- if (oldOrder && ord && this.manager.accountant) {
4540
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
4541
- }
4542
- if (ord) {
4543
- const updatedSlot = { ...ord, size: ctx.updateInfo.newSize };
4544
- // Update rawOnChain cache with new integers
4545
- if (ctx.finalInts) {
4546
- updatedSlot.rawOnChain = {
4547
- id: ord.orderId,
4548
- for_sale: String(ctx.finalInts.sell),
4549
- sell_price: {
4550
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4551
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4552
- }
4553
- };
4554
- }
4555
- updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
4556
- }
4557
- this.manager.logger.log(`Size update complete: ${ctx.updateInfo.partialOrder.orderId}`, 'info');
4558
- updateOperationCount++;
4559
- }
4560
- else if (ctx.kind === 'create') {
4561
- const chainOrderId = res && res[1];
4562
- if (chainOrderId) {
4563
- // synchronizeWithChain handles the full VIRTUAL -> ACTIVE transition
4564
- // including orderId assignment and fee deduction.
4565
- await this.manager.synchronizeWithChain({
4566
- gridOrderId: ctx.order.id, chainOrderId, expectedType: ctx.order.type, fee: btsFeeData.createFee
4567
- }, 'createOrder');
4568
- // After sync, apply rawOnChain metadata if available
4569
- if (ctx.finalInts) {
4570
- const syncedOrder = this.manager.orders.get(ctx.order.id);
4571
- if (syncedOrder) {
4572
- updatesToApply.push({
4573
- order: {
4574
- ...syncedOrder,
4575
- rawOnChain: {
4576
- id: chainOrderId,
4577
- for_sale: String(ctx.finalInts.sell),
4578
- sell_price: {
4579
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4580
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4581
- }
4582
- }
4583
- },
4584
- context: 'post-placement-metadata'
4585
- });
4586
- }
4587
- }
4588
- this.manager.logger.log(`Placed ${ctx.order.type} order ${ctx.order.id} -> ${chainOrderId}`, 'info');
4589
- }
4590
- else {
4591
- const fingerprint = [
4592
- `type=${ctx.order.type || 'unknown'}`,
4593
- `price=${Format.formatPrice6(ctx.order.price)}`,
4594
- `size=${Format.formatAmount(ctx.order.size)}`
4595
- ].join(',');
4596
- this.manager.logger.log(`[COW] CRITICAL: Create op for slot ${ctx.order.id} (type=${ctx.order.type}) ` +
4597
- `returned no chainOrderId. Identify any orphaned on-chain order by local fingerprint ` +
4598
- `${fingerprint} before cancelling.`, 'error');
4599
- }
4600
- }
4601
- else if (ctx.kind === 'rotation') {
4602
- hadRotation = true;
4603
- const { rotation } = ctx;
4604
- const { oldOrder, newPrice, newGridId, newSize, type } = rotation;
4605
- if (!newGridId) {
4606
- // Size correction only
4607
- const ord = this.manager.orders.get(oldOrder.id || rotation.id);
4608
- // Apply optimistic accounting from pre-commit -> committed state,
4609
- // including blockchain update fee deduction.
4610
- if (oldOrder && ord && this.manager.accountant) {
4611
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData.updateFee, false);
4612
- }
4613
- if (ord) {
4614
- const updatedSlot = { ...ord, size: newSize };
4615
- // Update rawOnChain cache with new integers
4616
- if (ctx.finalInts) {
4617
- updatedSlot.rawOnChain = {
4618
- id: ord.orderId,
4619
- for_sale: String(ctx.finalInts.sell),
4620
- sell_price: {
4621
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4622
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4623
- }
4624
- };
4625
- }
4626
- updatesToApply.push({ order: updatedSlot, context: 'post-update-metadata' });
4627
- }
4628
- updateOperationCount++;
4629
- continue;
4630
- }
4631
- // Full rotation: old slot was virtualized in the committed working grid.
4632
- // Activate the destination slot with the existing on-chain orderId.
4633
- const slot = this.manager.orders.get(newGridId);
4634
- if (!slot) {
4635
- this.manager.logger.log(`[ROTATION] Destination slot ${newGridId} missing from master grid after COW commit - skipping activation, sync will reconcile`, 'error');
4636
- // Still clear source orderId to prevent a stale chain reference persisting
4637
- if (oldOrder?.id && oldOrder.id !== newGridId) {
4638
- const staleSource = this.manager.orders.get(oldOrder.id);
4639
- if (staleSource?.orderId) {
4640
- updatesToApply.push({
4641
- order: { ...staleSource, state: ORDER_STATES.VIRTUAL, orderId: null, rawOnChain: null },
4642
- context: 'post-rotation-source-clear'
4643
- });
4644
- }
4645
- }
4646
- continue;
4647
- }
4648
- const updatedSlot = {
4649
- ...slot,
4650
- id: newGridId,
4651
- type,
4652
- size: newSize,
4653
- price: newPrice,
4654
- state: ORDER_STATES.ACTIVE,
4655
- orderId: oldOrder?.orderId || slot.orderId || null
4656
- };
4657
- if (ctx.finalInts) {
4658
- updatedSlot.rawOnChain = {
4659
- id: updatedSlot.orderId,
4660
- for_sale: String(ctx.finalInts.sell),
4661
- sell_price: {
4662
- base: { amount: String(ctx.finalInts.sell), asset_id: ctx.finalInts.sellAssetId },
4663
- quote: { amount: String(ctx.finalInts.receive), asset_id: ctx.finalInts.receiveAssetId }
4664
- }
4665
- };
4666
- }
4667
- if (oldOrder && updatedSlot && this.manager.accountant) {
4668
- await this.manager.accountant.updateOptimisticFreeBalance(oldOrder, updatedSlot, 'order-update', btsFeeData.updateFee, false);
4669
- }
4670
- // Ensure rotation source slot is cleared in master state.
4671
- // COW projection may keep source slots ACTIVE when target keeps a non-zero
4672
- // virtual size, but after a successful on-chain rotation the source orderId
4673
- // must no longer remain attached to the old slot.
4674
- if (oldOrder?.id && oldOrder.id !== newGridId) {
4675
- const currentSource = this.manager.orders.get(oldOrder.id);
4676
- if (currentSource && currentSource.orderId) {
4677
- updatesToApply.push({
4678
- order: {
4679
- ...currentSource,
4680
- state: ORDER_STATES.VIRTUAL,
4681
- orderId: null,
4682
- rawOnChain: null
4683
- },
4684
- context: 'post-rotation-source-clear'
4685
- });
4686
- }
4687
- }
4688
- updatesToApply.push({ order: updatedSlot, context: 'post-rotation-metadata' });
4689
- }
4690
- }
4691
- // Apply all collected updates in a single batch
4692
- if (updatesToApply.length > 0) {
4693
- await this.manager.applyGridUpdateBatch(updatesToApply.map(u => u.order), 'batch-results-process', { skipAccounting: true });
4694
- }
4695
- return {
4696
- executed: true,
4697
- hadRotation,
4698
- updateOperationCount
4699
- };
1398
+ return cowRuntime.processBatchResults(this, result, opContexts);
4700
1399
  }
4701
1400
  /**
4702
1401
  * Perform grid recalculation triggered by trigger file.
@@ -4937,89 +1636,17 @@ class DEXBot {
4937
1636
  }
4938
1637
  }
4939
1638
  async requestGridReset(reason = 'structural change', options = {}) {
4940
- if (!this.manager || typeof this._performGridResync !== 'function') {
4941
- return { skipped: true, reason: 'grid resync unavailable' };
4942
- }
4943
- const message = reason ? `[CR-RESET] ${reason}` : '[CR-RESET] grid reset requested';
4944
- this._log(`${message}; rebuilding grid from fresh on-chain state`, 'info');
4945
- const resetOptions = {
4946
- ...options,
4947
- refreshCenterPrice: options.refreshCenterPrice !== false,
4948
- };
4949
- // Skip acquire if no lock exists (no fill processing to serialize against)
4950
- // or if the caller already holds the lock (re-entrant). In the latter case
4951
- // the lock's isReentrant() check prevents queueing and runs inline.
4952
- // Otherwise, wait on the queue — this is the intended path for callers
4953
- // outside the fill-processing context that need exclusive access.
4954
- if (!this.manager._fillProcessingLock || this.manager._fillProcessingLock.isReentrant()) {
4955
- return this._performGridResync(resetOptions);
4956
- }
4957
- return this.manager._fillProcessingLock.acquire(async () => this._performGridResync(resetOptions));
1639
+ return DexbotMaintenanceRuntime.requestGridReset(this, reason, options);
4958
1640
  }
4959
1641
  _wireStructuralGridResyncRequest() {
4960
- if (!this.manager || this.manager.requestStructuralGridResync)
4961
- return;
4962
- this.manager.requestStructuralGridResync = async (reason = 'structural recovery', details = {}) => {
4963
- if (this._shuttingDown) {
4964
- return { skipped: true, reason: 'shutting down' };
4965
- }
4966
- if (this._structuralGridResyncRunning || this._structuralGridResyncTimer) {
4967
- return { skipped: true, reason: 'structural grid resync already scheduled' };
4968
- }
4969
- const unmatchedCount = Array.isArray(details?.unmatchedChainOrders)
4970
- ? details.unmatchedChainOrders.length
4971
- : 0;
4972
- this._structuralGridResyncTimer = setTimeout(async () => {
4973
- this._structuralGridResyncTimer = null;
4974
- if (this._shuttingDown)
4975
- return;
4976
- this._structuralGridResyncRunning = true;
4977
- try {
4978
- // Try the lighter persisted-grid reload before full reset.
4979
- const persistedResult = await this._recoverFromPersistedGrid();
4980
- if (persistedResult.success) {
4981
- if (this.manager?._recoveryState) {
4982
- this.manager._recoveryState = { ...this.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
4983
- }
4984
- return;
4985
- }
4986
- const suffix = unmatchedCount > 0 ? ` (${unmatchedCount} unmatched chain order(s))` : '';
4987
- this._warn(`[RECOVERY] Running structural full grid resync for ${reason}${suffix}`);
4988
- const resetResult = await this.requestGridReset('rms_structural_grid_resync', {
4989
- refreshCenterPrice: true,
4990
- });
4991
- if (resetResult && this.manager?._recoveryState) {
4992
- this.manager._recoveryState = { ...this.manager._recoveryState, attemptCount: 0, lastAttemptAt: 0, lastFailureAt: 0 };
4993
- }
4994
- }
4995
- catch (err) {
4996
- this._warn(`[RECOVERY] Structural full grid resync failed: ${err.message}`);
4997
- }
4998
- finally {
4999
- this._structuralGridResyncRunning = false;
5000
- if (this.manager?._recoveryState) {
5001
- this.manager._recoveryState = { ...this.manager._recoveryState, structuralResyncRequested: false };
5002
- }
5003
- }
5004
- }, 0);
5005
- return { scheduled: true };
5006
- };
1642
+ return DexbotMaintenanceRuntime.wireStructuralGridResyncRequest(this);
5007
1643
  }
5008
1644
  /**
5009
1645
  * Get current metrics for monitoring and debugging.
5010
1646
  * @returns {Object} Metrics snapshot
5011
1647
  */
5012
1648
  getMetrics() {
5013
- this.manager?._cleanExpiredLocks?.();
5014
- return {
5015
- ...this._metrics,
5016
- queueDepth: this._incomingFillQueue.length,
5017
- fillProcessingLockActive: this.manager?._fillProcessingLock?.isLocked() || false,
5018
- divergenceLockActive: this.manager?._divergenceLock?.isLocked() || false,
5019
- shadowLocksActive: this.manager?.shadowOrderIds?.size || 0,
5020
- recoveryExhaustedAt: this.manager?._recoveryExhaustedAt || null,
5021
- recentFillsTracked: this._recentlyProcessedFills.size
5022
- };
1649
+ return DexbotMaintenanceRuntime.getMetrics(this);
5023
1650
  }
5024
1651
  /**
5025
1652
  * Execute grid maintenance checks in strict order with pipeline consensus.
@@ -5074,41 +1701,7 @@ class DEXBot {
5074
1701
  * @private
5075
1702
  */
5076
1703
  async _runDustHealthCheck() {
5077
- if (this._shuttingDown || !this.manager)
5078
- return;
5079
- try {
5080
- const health = await this.manager.checkGridHealth(this.updateOrdersOnChainPlan?.bind(this));
5081
- const buyDust = health.buyDustOrders || [];
5082
- const sellDust = health.sellDustOrders || [];
5083
- const totalDust = buyDust.length + sellDust.length;
5084
- if (totalDust > 0) {
5085
- this._log(`[DUST] Health check: ${totalDust} dust order(s) (buy=${buyDust.length}, sell=${sellDust.length})`);
5086
- const lock = this.manager._fillProcessingLock;
5087
- if (lock && typeof lock.acquire === 'function') {
5088
- await lock.acquire(async () => {
5089
- await this._cancelDustOrders({
5090
- buy: health.buyDustOrders,
5091
- sell: health.sellDustOrders,
5092
- });
5093
- }, { timeout: TIMING.DUST_CANCEL_TIMEOUT_MS });
5094
- }
5095
- else {
5096
- this._warn('[DUST] Fill lock unavailable — cancelling dust without lock (potential race)');
5097
- await this._cancelDustOrders({
5098
- buy: health.buyDustOrders,
5099
- sell: health.sellDustOrders,
5100
- });
5101
- }
5102
- }
5103
- }
5104
- catch (err) {
5105
- if (err?.message?.includes('Lock acquisition timeout')) {
5106
- this._warn('[DUST] Lock busy, skipping dust cancel this cycle (retry in 5 min)');
5107
- }
5108
- else {
5109
- this._warn(`[DUST] Health check error (retry in 5 min): ${err?.message || err}`);
5110
- }
5111
- }
1704
+ return DexbotMaintenanceRuntime.runDustHealthCheck(this);
5112
1705
  }
5113
1706
  /**
5114
1707
  * Set up the periodic dust health check interval.
@@ -5118,12 +1711,7 @@ class DEXBot {
5118
1711
  * @private
5119
1712
  */
5120
1713
  _setupDustHealthCheckInterval() {
5121
- this._dustHealthCheckTimer = setInterval(() => {
5122
- this._runDustHealthCheck();
5123
- }, TIMING.DUST_HEALTH_CHECK_INTERVAL_MS);
5124
- if (typeof this._dustHealthCheckTimer?.unref === 'function') {
5125
- this._dustHealthCheckTimer.unref();
5126
- }
1714
+ return DexbotMaintenanceRuntime.setupDustHealthCheckInterval(this);
5127
1715
  }
5128
1716
  /**
5129
1717
  * Perform grid maintenance: fund thresholds, spread condition, grid health, divergence.