dexbot 1.2.2 → 1.2.4

package/dist/modules/dexbot_class.js

@@ -596,6 +596,42 @@ class DEXBot {
             }
             this.manager.logger.log(`[RECOVERY] Grid reloaded from persisted snapshot: ${this.manager.orders.size} orders, ` +
                 `${chainOpenOrders.length} on-chain orders synced`, 'info');
+            // Gap 2: Check for unmatched chain orders after reload + sync.
+            // If the sync resolved all unmatched entries, _lastUnmatchedChainOrders
+            // was cleared by the sync engine. If any remain, the persisted snapshot
+            // produced an inconsistent grid — reject so the structural resync
+            // falls through to requestGridReset (full rebuild from chain).
+            const remainingUnmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
+                ? this.manager._lastUnmatchedChainOrders
+                : [];
+            if (remainingUnmatched.length > 0) {
+                const sample = remainingUnmatched.slice(0, 3)
+                    .map(o => this._formatUnmatchedChainOrderForLog(o))
+                    .join(' | ');
+                this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but ${remainingUnmatched.length} unmatched chain order(s) ` +
+                    `remain${sample ? ` (${sample})` : ''}. Rejecting — full grid reset required.`, 'warn');
+                return { success: false, reason: `grid inconsistent after reload: ${remainingUnmatched.length} unmatched remain` };
+            }
+            // If the reloaded grid is still bloated, reject recovery so the
+            // caller falls through to a full grid reset (requestGridReset).
+            // Without this check, a bloated snapshot gets accepted as "success"
+            // and the structural-resync loop loads the same broken state forever.
+            //
+            // NOTE: loadGrid() already fires requestStructuralGridResync when it
+            // detects bloat internally, so the inner async resync may be in flight
+            // by the time this outer check runs. That's fine — the structural-resync
+            // gate (_structuralGridResyncRunning / _structuralGridResyncTimer) dedup's
+            // concurrent requests. This outer check exists so the synchronous return
+            // value is honest about the state; the inner resync is a safety net.
+            const { isGridBloated } = require('./order/grid');
+            const ordersArr = Array.from(this.manager.orders.values());
+            const bloatPostRecovery = isGridBloated(this.manager, ordersArr);
+            if (bloatPostRecovery.bloated) {
+                const d = bloatPostRecovery.details;
+                this.manager.logger.log(`[RECOVERY] Persisted grid reloaded but still bloated ` +
+                    `(${d.gridSize} slots, max ${d.maxAllowed}). Rejecting — full grid reset required.`, 'warn');
+                return { success: false, reason: 'grid still bloated after reload' };
+            }
             return { success: true };
         }
         catch (err) {
@@ -2794,6 +2830,45 @@ class DEXBot {
             this.manager.logger.log(`[COW][UNCERTAIN] Discarded planned CREATEs (no chain match after ${recheckRound} recheck(s)): ${discarded
                 .map(d => d.slotId)
                 .join(', ')}`, 'warn');
+            // Restore target grid sizes for discarded CREATEs so the slots are
+            // not left as virtual/0 after recovery. Without this, the slot stays
+            // at size 0 (set by the fill handler) and is never reactivated until
+            // a fresh fill cycle triggers another rebalance that happens to succeed.
+            // entry.order is the pending-broadcast target order (captured at broadcast
+            // time); its .id matches entry.slotId — the lookup below keys on slotId.
+            for (const entry of discarded) {
+                if (entry.order && entry.slotId) {
+                    const current = this.manager.orders.get(entry.slotId);
+                    if (current && current.state === ORDER_STATES.VIRTUAL && !current.orderId) {
+                        try {
+                            await this.manager._applyOrderUpdate({ ...entry.order, state: ORDER_STATES.VIRTUAL, orderId: null }, 'uncertain-recovery-restore-size', { skipAccounting: true, fee: 0 });
+                            this.manager.logger.log(`[COW][UNCERTAIN] Restored target size for discarded CREATE slot ${entry.slotId} (size: ${entry.order.size})`, 'info');
+                        }
+                        catch (restoreErr) {
+                            this.manager.logger.log(`[COW][UNCERTAIN] Failed to restore size for slot ${entry.slotId}: ${restoreErr?.message || restoreErr}`, 'warn');
+                        }
+                    }
+                }
+            }
+        }
+        // ---- Structural resync safeguard after skipAccounting restore ----
+        // The discarded CREATE restore above used skipAccounting: true to avoid
+        // double-counting when the structural resync recalculates accounting
+        // from scratch. Ensure one is scheduled — if already in-flight (timer
+        // or running), the existing resync will handle the accounting fix.
+        if (discarded.length > 0 && typeof this.manager?.requestStructuralGridResync === 'function') {
+            const alreadyScheduled = this._structuralGridResyncRunning || this._structuralGridResyncTimer;
+            if (!alreadyScheduled) {
+                this.manager.logger.log(`[COW][UNCERTAIN] Scheduling structural resync after discarded CREATE restore ` +
+                    `(skipAccounting used — resync needed for fund recalculation)`, 'info');
+                this.manager.requestStructuralGridResync('cow-uncertain-accounting-repair', { discardedCount: discarded.length }).catch((err) => {
+                    this.manager.logger.log(`[COW][UNCERTAIN] Failed to schedule accounting repair resync: ${err.message}`, 'error');
+                });
+            }
+            else {
+                this.manager.logger.log(`[COW][UNCERTAIN] Structural resync already ${this._structuralGridResyncRunning ? 'running' : 'scheduled'}; ` +
+                    `it will repair accounting after discarded CREATE restore`, 'debug');
+            }
         }
         this._clearPendingBroadcasts();
         // Post-recovery safety net: if there are still unmatched chain
@@ -2861,18 +2936,22 @@ class DEXBot {
         return { executed: false, hadRotation, uncertain: true, adopted, discarded };
     }
     /**
-     * Auto-cancel a single unmatched chain order from the recovery snapshot.
+     * Auto-cancel a price-drift orphan from the unmatched-order snapshot.
+     *
+     * Only cancels entries with reason === 'price-drift-orphan' — these are
+     * surplus orders that drifted away from their slot price and have no
+     * adoptable grid slot. All other unmatched orders (duplicate-price-level,
+     * already-matched-slot, etc.) are adoptable positions that the structural
+     * resync will integrate into the grid; cancelling them destroys capital.
      *
      * This is the post-recovery safety net: if, after
-     * _reconcileAfterUncertainBroadcast runs, there are still chain orders
-     * the bot doesn't recognize (e.g. from a network partition, or from a
-     * daemon timeout that we couldn't even fingerprint), we cancel ONE of
-     * them per call. Per-cycle cap = 1 — the next cycle will pick up the
-     * next unmatched order if more remain.
+     * _reconcileAfterUncertainBroadcast runs, there are still price-drift
+     * orphans, cancel ONE per cycle. Per-cycle cap = 1 (or 5 in recovery mode)
+     * — the next cycle will pick up the next orphan if more remain.
      *
      * Safety conditions (ALL must hold):
      *   1. _pendingBroadcasts is empty (no in-flight recovery)
-     *   2. _lastUnmatchedChainOrders is non-empty
+     *   2. _lastUnmatchedChainOrders contains at least one price-drift-orphan
      *   3. The current cycle has not already auto-cancelled an orphan
      *      (tracked via this._autoCancelOrphanCycleMarker)
      *
@@ -2908,17 +2987,27 @@ class DEXBot {
         if (unmatched.length === 0) {
             return { cancelled: false, reason: 'no-unmatched' };
         }
-        const priceDriftOrphan = unmatched.find(u => u && u.reason === 'price-drift-orphan');
-        const target = priceDriftOrphan || unmatched[0];
-        const orderId = target?.id || target?.orderId || target?.chainOrderId;
+        // Check for fingerprinted entries first — these came from a pending
+        // broadcast (missing-create-result) and must be handled by the recovery
+        // path, not by auto-cancel. The first fingerprinted entry is checked
+        // regardless of its reason field.
+        const fingerprinted = unmatched.find(u => u && u.fingerprint);
+        if (fingerprinted) {
+            return { cancelled: false, reason: 'fingerprinted-handle-via-recovery' };
+        }
+        // Only cancel price-drift orphans — these are surplus orders that
+        // drifted away from their assigned slot price and have no adoptable
+        // grid slot. All other unmatched orders (duplicate-price-level,
+        // already-matched-slot, etc.) are adoptable positions that the structural
+        // resync will integrate into the grid — cancelling them destroys capital.
+        const target = unmatched.find(u => u && u.reason === 'price-drift-orphan');
+        if (!target) {
+            return { cancelled: false, reason: 'no-price-drift-orphan', message: 'no price-drift orphan to cancel; other unmatched orders are adoptable' };
+        }
+        const orderId = target.id || target.orderId || target.chainOrderId;
         if (!orderId) {
             return { cancelled: false, reason: 'no-orderId' };
         }
-        if (target?.fingerprint) {
-            // Fingerprinted unmatched orders came from a pending broadcast.
-            // The recovery path is the right place to handle them, not here.
-            return { cancelled: false, reason: 'fingerprinted-handle-via-recovery' };
-        }
         if (!chainOrders?.cancelOrder) {
             return { cancelled: false, reason: 'cancelOrder-unavailable' };
         }
@@ -3776,111 +3865,112 @@ class DEXBot {
             ? Array.from(this.manager._pendingBroadcasts.values())
             : [];
         if (hasCreateActions && (unmatchedChainOrders.length > 0 || pendingBroadcasts.length > 0)) {
-            // ---- FIX 3: Absorb unmatched orders instead of rejecting ----
-            // When unmatched chain orders exist, try to auto-cancel them
-            // inline to unblock the CREATE batch. Only fall through to
-            // full rejection + structural resync if cancellation fails.
-            let cancelledUnmatched = 0;
-            if (unmatchedChainOrders.length > 0 && pendingBroadcasts.length === 0) {
-                // Bounded-parallel cancellation: process unmatched orders in
-                // concurrent batches (3 at a time) to avoid serialising 19+
-                // individual cancel txs (~0.5s each = 10s batch hold).
-                const CANCEL_CONCURRENCY = 3;
-                const cancelable = unmatchedChainOrders.filter(u => {
-                    const oid = u?.id || u?.orderId || u?.chainOrderId;
-                    return Boolean(oid) && !u?.fingerprint;
-                });
-                for (let i = 0; i < cancelable.length; i += CANCEL_CONCURRENCY) {
-                    const batch = cancelable.slice(i, i + CANCEL_CONCURRENCY);
-                    const results = await Promise.allSettled(batch.map(async (unmatched) => {
-                        const orderId = unmatched.id || unmatched.orderId || unmatched.chainOrderId;
-                        this.manager.logger.log(`[COW] Auto-cancelling unmatched chain order ${orderId} ` +
-                            `(${this._formatUnmatchedChainOrderForLog(unmatched)}) to unblock CREATE batch.`, 'warn');
-                        await chainOrders.cancelOrder(this.account, this.privateKey, orderId);
-                        if (typeof chainOrders.recordOwnCancel === 'function') {
-                            chainOrders.recordOwnCancel(orderId);
-                        }
-                    }));
-                    for (const r of results) {
-                        if (r.status === 'fulfilled')
-                            cancelledUnmatched++;
-                        else
-                            this.manager.logger.log(`[COW] Failed to cancel unmatched chain order: ${r.reason?.message || r.reason}`, 'warn');
-                    }
-                }
-                if (cancelable.length > 0 && cancelledUnmatched >= cancelable.length) {
-                    this.manager.logger.log(`[COW] Cancelled all ${cancelledUnmatched} unmatched chain order(s); proceeding with CREATE batch.`, 'warn');
-                    // Optimistic clear: the cancel txs are sent but not yet
-                    // confirmed. If a cancel silently fails on chain, the
-                    // next sync cycle will re-detect the unmatched order and
-                    // re-enter this guard — that's safe because the cancel
-                    // is idempotent and the re-detection is non-blocking.
-                    this.manager._lastUnmatchedChainOrders = [];
-                }
-                else if (cancelable.length > 0) {
-                    this.manager.logger.log(`[COW] Cancelled ${cancelledUnmatched}/${cancelable.length} unmatched chain order(s); ` +
-                        `remaining must be handled by structural reconciliation.`, 'warn');
-                }
-            }
-            // Re-check after auto-cancel attempt
-            const remainingUnmatched = Array.isArray(this.manager?._lastUnmatchedChainOrders)
-                ? this.manager._lastUnmatchedChainOrders
-                : [];
-            if (remainingUnmatched.length > 0 || pendingBroadcasts.length > 0) {
-                const blockers = [];
-                if (remainingUnmatched.length > 0)
-                    blockers.push(`${remainingUnmatched.length} unmatched chain order(s)`);
-                if (pendingBroadcasts.length > 0)
-                    blockers.push(`${pendingBroadcasts.length} pending broadcast(s)`);
-                const reasonText = blockers.join(' and ');
-                if (pendingBroadcasts.length > 0) {
-                    this.manager.logger.log(`[COW] Rejecting CREATE batch: ${reasonText} from a prior uncertain ` +
-                        `broadcast. Running recovery before placing replacement orders.`, 'error');
-                }
-                else {
-                    const sample = remainingUnmatched
-                        .slice(0, 3)
-                        .map(order => this._formatUnmatchedChainOrderForLog(order))
-                        .join(' | ');
-                    this.manager.logger.log(`[COW] Rejecting CREATE batch: ${reasonText} ` +
-                        `are not represented in the grid${sample ? ` (${sample})` : ''}. ` +
-                        `Run structural reconciliation before placing replacement orders.`, 'error');
-                }
+            // ---- Handle pending broadcasts first ----
+            if (pendingBroadcasts.length > 0) {
+                this.manager.logger.log(`[COW] Rejecting CREATE batch: ${pendingBroadcasts.length} pending broadcast(s) from a prior uncertain ` +
+                    `broadcast. Running recovery before placing replacement orders.`, 'error');
                 if (typeof this.manager.requestStructuralGridResync === 'function') {
                     if (this.manager._recoveryState)
                         this.manager._recoveryState.structuralResyncRequested = true;
-                    await this.manager.requestStructuralGridResync(pendingBroadcasts.length > 0
-                        ? 'pending broadcasts before COW create'
-                        : 'unmatched chain orders before COW create', pendingBroadcasts.length > 0
-                        ? { pendingBroadcasts: pendingBroadcasts.map(p => p.slotId) }
-                        : { unmatchedChainOrders: remainingUnmatched });
+                    await this.manager.requestStructuralGridResync('pending broadcasts before COW create', { pendingBroadcasts: pendingBroadcasts.map(p => p.slotId) });
                 }
-                // If we have pending broadcasts, drive the recovery now so the
-                // next planning cycle has a clean state.
-                if (pendingBroadcasts.length > 0) {
-                    try {
-                        await this._reconcileAfterUncertainBroadcast(new BroadcastUncertainError('rejected CREATE batch had pending broadcasts', {
-                            operations: pendingBroadcasts.map(p => p.order),
-                            accountName: this.account,
-                            batchId: this._currentBatchId || null,
-                            payload: null,
-                            timeoutMs: null
-                        }), []);
-                    }
-                    catch (recoverErr) {
-                        this.manager.logger.log(`[COW] Recovery from pending broadcasts failed: ${recoverErr?.message || recoverErr}`, 'error');
-                    }
+                try {
+                    await this._reconcileAfterUncertainBroadcast(new BroadcastUncertainError('rejected CREATE batch had pending broadcasts', {
+                        operations: pendingBroadcasts.map(p => p.order),
+                        accountName: this.account,
+                        batchId: this._currentBatchId || null,
+                        payload: null,
+                        timeoutMs: null
+                    }), []);
+                }
+                catch (recoverErr) {
+                    this.manager.logger.log(`[COW] Recovery from pending broadcasts failed: ${recoverErr?.message || recoverErr}`, 'error');
                 }
                 return {
                     executed: false,
                     aborted: true,
-                    reason: pendingBroadcasts.length > 0 ? 'PENDING_BROADCASTS' : 'UNMATCHED_CHAIN_ORDERS',
-                    unmatchedChainOrders: pendingBroadcasts.length > 0 ? [] : remainingUnmatched,
-                    pendingBroadcasts: pendingBroadcasts.map(p => p.slotId),
+                    reason: 'PENDING_BROADCASTS',
                     hadRotation: false
                 };
             }
-            // All unmatched were cancelled — fall through to continue batch
+            // ---- Unmatched orders: adopt instead of cancel ----
+            // Unmatched orders are legitimate on-chain positions the grid
+            // hasn't adopted yet. Cancelling them destroys capital and
+            // creates gaps in the order book. Instead, re-sync to adopt
+            // them into the grid, then reject (the sync invalidated the
+            // working grid, so the existing COW plan is stale).
+            // NOTE: syncFromOpenOrders may be a partial no-op under certain
+            // lock conditions (e.g. _fillProcessingLock + !isReentrant).
+            // Structural resync scheduled below handles adoption regardless.
+            const unmatchedSample = unmatchedChainOrders
+                .slice(0, 3)
+                .map(o => this._formatUnmatchedChainOrderForLog(o))
+                .join(' | ');
+            this.manager.logger.log(`[COW] ${unmatchedChainOrders.length} unmatched chain order(s) blocking CREATES ` +
+                (unmatchedSample ? `(${unmatchedSample})` : '') +
+                ` — adopting via sync instead of cancelling`, 'info');
+            try {
+                const accountRef = this.account;
+                const freshSnapshot = await chainOrders.readOpenOrders(accountRef);
+                if (freshSnapshot && freshSnapshot.length > 0) {
+                    const syncResult = await this.manager.syncFromOpenOrders(freshSnapshot, {
+                        skipAccounting: true,
+                    });
+                    // Only overwrite _lastUnmatchedChainOrders when sync actually
+                    // processed orders (filled + updated + corrected > 0).
+                    // Force-release and lock-contention early-exit paths return
+                    // empty arrays without touching _lastUnmatchedChainOrders —
+                    // overwriting with [] would drop stale unmatched entries.
+                    if (syncResult && Array.isArray(syncResult.unmatchedChainOrders)) {
+                        const processed = (syncResult.filledOrders?.length || 0) +
+                            (syncResult.updatedOrders?.length || 0) +
+                            (syncResult.ordersNeedingCorrection?.length || 0);
+                        if (processed > 0) {
+                            this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders;
+                            this.manager.logger.log(`[COW] Adopted chain order(s) via sync: ${processed} processed, ` +
+                                `${syncResult.unmatchedChainOrders.length} still unmatched`, 'info');
+                        }
+                        else {
+                            // processed === 0 with unmatched still present means the
+                            // sync could not adopt the unmatched chain orders. This is
+                            // normal when called re-entrantly (inside _fillProcessingLock):
+                            // the sync engine runs inline and either timed out (unlikely
+                            // for a re-entrant call) or all chain orders were already
+                            // matched — leaving only stale unmatched entries.
+                            const syncUnmatchedCount = syncResult.unmatchedChainOrders.length;
+                            this.manager.logger.log(`[COW] Sync returned without processing (processed=0, ` +
+                                `unmatched=${syncUnmatchedCount} in result, ` +
+                                `_lastUnmatchedChainOrders=${unmatchedChainOrders.length}). ` +
+                                `Structural resync will handle adoption.`, syncUnmatchedCount > 0 ? 'warn' : 'debug');
+                            // If the sync result itself has unmatched entries that
+                            // differ from _lastUnmatchedChainOrders, adopt them now
+                            // so the stale tracker is more accurate for the resync.
+                            if (syncUnmatchedCount > 0 && syncUnmatchedCount !== unmatchedChainOrders.length) {
+                                this.manager._lastUnmatchedChainOrders = syncResult.unmatchedChainOrders.map((o) => ({ ...o }));
+                                this.manager.logger.log(`[COW] Updated _lastUnmatchedChainOrders from sync result: ` +
+                                    `${unmatchedChainOrders.length} → ${syncUnmatchedCount}`, 'debug');
+                            }
+                        }
+                    }
+                }
+            }
+            catch (syncErr) {
+                this.manager.logger.log(`[COW] Failed to sync/unmatched orders: ${syncErr?.message || syncErr}`, 'warn');
+            }
+            // Working grid is stale after master grid mutation from sync.
+            // Request structural resync to rebuild the grid on the next cycle.
+            if (typeof this.manager.requestStructuralGridResync === 'function') {
+                if (this.manager._recoveryState)
+                    this.manager._recoveryState.structuralResyncRequested = true;
+                await this.manager.requestStructuralGridResync('unmatched chain orders before COW create', { unmatchedChainOrders: unmatchedChainOrders });
+            }
+            this.manager.logger.log(`[COW] Rejecting CREATE batch after sync: working grid invalidated by master mutation`, 'info');
+            return {
+                executed: false,
+                aborted: true,
+                reason: 'UNMATCHED_CHAIN_ORDERS',
+                hadRotation: false
+            };
         }
         const { assetA, assetB } = this.manager.assets;
         const operations = [];