dexbot 1.4.0 → 1.4.2

package/dist/modules/order/grid.js

@@ -499,7 +499,7 @@ async function loadGrid(manager, grid, boundaryIdx = null) {
         return;
     return await manager._gridLock.acquire(async () => {
         try {
-            await manager._initializeAssets();
+            await (0, system_1.withBlockchainRetry)(() => manager._initializeAssets(), 'initializeAssets', { logger: manager.logger });
         }
         catch (e) {
             manager.logger?.log?.(`Asset initialization failed during grid load: ${(0, errors_1.getErrorMessage)(e)}`, 'warn');
@@ -596,7 +596,12 @@ async function loadGrid(manager, grid, boundaryIdx = null) {
 async function initializeGrid(manager) {
     if (!manager)
         throw new Error('initializeGrid requires a manager instance');
-    await manager._initializeAssets();
+    try {
+        await (0, system_1.withBlockchainRetry)(() => manager._initializeAssets(), 'initializeAssets', { logger: manager.logger });
+    }
+    catch (e) {
+        manager.logger?.log?.(`Asset initialization failed during grid init: ${(0, errors_1.getErrorMessage)(e)}`, 'warn');
+    }
     // FIX: Add explicit state validation to prevent cryptic errors later
     if (!manager.assets || !manager.assets.assetA || !manager.assets.assetB) {
         throw new Error('Asset initialization did not complete properly - assetA or assetB undefined');
@@ -867,49 +872,82 @@ async function recalculateGrid(manager, opts) {
     const { readOpenOrdersFn, chainOrders, account, privateKey } = opts;
     // Suppress invariant warnings during full resync
     manager.startBootstrap();
-    try {
-        // FIX: Use consistent optional chaining pattern for logger calls
-        manager.logger?.log?.('Starting full resync...', 'info');
-        await manager._initializeAssets();
-        await manager.fetchAccountTotals();
-        const READ_OPEN_ORDERS_TIMEOUT_MS = 30000;
-        let _readOpenOrdersTimer;
-        let chainOpenOrders;
+    // Total timeout across all steps — prevents indefinite hang even if
+    // an individual withBlockchainRetry step pins the event loop.
+    const totalTimeoutMs = constants_1.PIPELINE_TIMING.TIMEOUT_MS * 2; // 10 min
+    let _resyncAborted = false;
+    const work = (async () => {
         try {
-            chainOpenOrders = await Promise.race([
-                readOpenOrdersFn(),
-                new Promise((_, reject) => {
-                    _readOpenOrdersTimer = setTimeout(() => reject(new Error('readOpenOrders timeout')), READ_OPEN_ORDERS_TIMEOUT_MS);
-                })
-            ]);
+            manager.logger?.log?.('Starting full resync...', 'info');
+            if (_resyncAborted)
+                return;
+            // #1: Initialize assets with timeout + retry + node failover
+            try {
+                await (0, system_1.withBlockchainRetry)(() => manager._initializeAssets(), 'initializeAssets', { logger: manager.logger });
+            }
+            catch (e) {
+                manager.logger?.log?.(`Asset initialization failed during resync: ${(0, errors_1.getErrorMessage)(e)}`, 'warn');
+            }
+            if (_resyncAborted)
+                return;
+            // #2: Fetch account totals with timeout + retry + node failover
+            await (0, system_1.withBlockchainRetry)(() => manager.fetchAccountTotals(), 'fetchAccountTotals', { logger: manager.logger });
+            if (_resyncAborted)
+                return;
+            // #3: Read open orders with timeout + retry + node failover
+            const chainOpenOrders = await (0, system_1.withBlockchainRetry)(() => readOpenOrdersFn(), 'readOpenOrders', { logger: manager.logger });
+            if (_resyncAborted)
+                return;
+            if (!Array.isArray(chainOpenOrders))
+                return;
+            await (0, system_1.withBlockchainRetry)(() => manager.syncFromOpenOrders(chainOpenOrders, { skipAccounting: true }), 'syncFromOpenOrders', { logger: manager.logger });
+            if (_resyncAborted)
+                return;
+            manager.resetFunds();
+            if (_resyncAborted)
+                return;
+            await manager.persistGrid();
+            if (_resyncAborted)
+                return;
+            await initializeGrid(manager);
+            if (_resyncAborted)
+                return;
+            const { reconcileGridOrders } = require('./grid_reconcile');
+            // #5: Reconcile grid orders with timeout + retry + node failover
+            try {
+                await (0, system_1.withBlockchainRetry)(() => reconcileGridOrders({ manager, config: manager.config, account, privateKey, chainOrders, chainOpenOrders }), 'reconcileGridOrders', 
+                // 5 min: Phase 2 of reconcile does sequential creates (~3s each);
+                // the default 30s timeout would kill mid-batch and cause duplicate-
+                // accumulation death spirals. PIPELINE_TIMING.TIMEOUT_MS gives enough
+                // headroom for all pending creates+updates to finish in one shot.
+                { logger: manager.logger, timeoutMs: constants_1.PIPELINE_TIMING.TIMEOUT_MS });
+            }
+            catch (err) {
+                manager.logger?.log?.(`Error during startup order reconciliation: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
+                throw new Error(`Grid recalculation failed during order reconciliation: ${(0, errors_1.getErrorMessage)(err)}`);
+            }
+            if (_resyncAborted)
+                return;
+            manager.logger?.log?.('Full resync complete.', 'info');
         }
         finally {
-            clearTimeout(_readOpenOrdersTimer);
+            manager.finishBootstrap();
         }
-        if (!Array.isArray(chainOpenOrders))
-            return;
-        // CRITICAL: Filter out PARTIAL orders before synchronizing - they're from old grid
-        // and shouldn't be part of the fresh regenerated grid structure
-        const activeOrders = chainOpenOrders.filter((o) => o.state !== constants_1.ORDER_STATES.PARTIAL);
-        await manager.syncFromOpenOrders(activeOrders, { skipAccounting: true });
-        manager.resetFunds();
-        await manager.persistGrid();
-        await initializeGrid(manager);
-        const { reconcileGridOrders } = require('./grid_reconcile');
-        // FIX: Add error context for debugging grid recalculation issues
-        try {
-            await reconcileGridOrders({ manager, config: manager.config, account, privateKey, chainOrders, chainOpenOrders });
-        }
-        catch (err) {
-            manager.logger?.log?.(`Error during startup order reconciliation: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
-            throw new Error(`Grid recalculation failed during order reconciliation: ${(0, errors_1.getErrorMessage)(err)}`);
-        }
-        // FIX: Use consistent optional chaining pattern for logger calls
-        manager.logger?.log?.('Full resync complete.', 'info');
-    }
-    finally {
-        manager.finishBootstrap();
-    }
+    })();
+    // Swallow late rejection if timeout wins the race
+    Promise.resolve(work).catch(() => { });
+    let timeoutId;
+    const result = await Promise.race([
+        work,
+        new Promise((_, reject) => {
+            timeoutId = setTimeout(() => {
+                _resyncAborted = true;
+                reject(new Error(`recalculateGrid timed out after ${totalTimeoutMs}ms`));
+            }, totalTimeoutMs);
+        })
+    ]);
+    clearTimeout(timeoutId);
+    return result;
 }
 /**
  * Check for grid divergence and trigger update if threshold is met.
@@ -1465,8 +1503,21 @@ async function checkSpreadCondition(manager, _BitShares, updateOrdersOnChainBatc
         correction = await prepareSpreadCorrectionOrders(manager, decision.side);
         if (!correction)
             return false;
-        const placeCount = correction.ordersToPlace?.length || 0;
-        const updateCount = correction.ordersToUpdate?.length || 0;
+        let placeCount = correction.ordersToPlace?.length || 0;
+        let updateCount = correction.ordersToUpdate?.length || 0;
+        // STARVATION FALLBACK: If the selected side has no correctable slots (e.g.
+        // all SPREAD slots already filled or misaligned), try the opposite side.
+        if ((placeCount + updateCount) === 0) {
+            const oppositeSide = decision.side === constants_1.ORDER_TYPES.BUY ? constants_1.ORDER_TYPES.SELL : constants_1.ORDER_TYPES.BUY;
+            manager.logger?.log?.(`[SPREAD] Side ${decision.side} produced zero candidates; ` +
+                `trying opposite side ${oppositeSide}.`, 'debug');
+            const oppositeCorrection = await prepareSpreadCorrectionOrders(manager, oppositeSide);
+            if (oppositeCorrection) {
+                correction = oppositeCorrection;
+                placeCount = correction.ordersToPlace?.length || 0;
+                updateCount = correction.ordersToUpdate?.length || 0;
+            }
+        }
         // Capture fund snapshot under lock for pre-flight verification before broadcast
         fundSnapshot = _snapshotFundState(manager);
         return (placeCount + updateCount) > 0;
@@ -1492,22 +1543,32 @@ async function checkSpreadCondition(manager, _BitShares, updateOrdersOnChainBatc
             // Instead of silently aborting, re-plan with fresh funds so the
             // correction still applies on this cycle.  The pre-flight check
             // still guards against placing orders based on stale fund snapshots.
-            const decision = determineOrderSideByFunds(manager, lastPrice);
-            if (decision.side) {
-                correction = await prepareSpreadCorrectionOrders(manager, decision.side);
-                if (correction && ((correction.ordersToPlace?.length || 0) + (correction.ordersToUpdate?.length || 0) > 0)) {
-                    fundSnapshot = currentFunds;
-                    manager.logger?.log?.(`[SPREAD] Fund state changed between lock release and broadcast — ` +
-                        `re-planned with updated funds: ${correction.ordersToPlace?.length || 0} creates, ` +
-                        `${correction.ordersToUpdate?.length || 0} updates`, 'info');
-                }
-                else {
-                    manager.logger?.log?.(`[SPREAD] Fund state changed; re-plan produced no viable orders. Skipping cycle.`, 'warn');
-                    return { ordersPlaced: 0, partialsMoved: 0 };
-                }
+            // Re-acquire _gridLock for the re-plan to ensure consistent grid
+            // state (the lock is re-entrant for this call chain — the outer
+            // acquire's callback completed before we reach here, so there is
+            // no nested lock to recurse into).  If determineOrderSideByFunds
+            // or prepareSpreadCorrectionOrders grow to hold the lock for
+            // heavy work, hoist the result to avoid serial re-execution.
+            const rePlanResult = await manager._gridLock.acquire(async () => {
+                const decision = determineOrderSideByFunds(manager, lastPrice);
+                if (!decision.side)
+                    return { side: false };
+                const c = await prepareSpreadCorrectionOrders(manager, decision.side);
+                return { side: true, correction: c };
+            });
+            if (!rePlanResult.side) {
+                manager.logger?.log?.(`[SPREAD] Fund state changed; no side has sufficient funds for re-plan. Skipping cycle.`, 'warn');
+                return { ordersPlaced: 0, partialsMoved: 0 };
+            }
+            correction = rePlanResult.correction;
+            if (correction && ((correction.ordersToPlace?.length || 0) + (correction.ordersToUpdate?.length || 0) > 0)) {
+                fundSnapshot = currentFunds;
+                manager.logger?.log?.(`[SPREAD] Fund state changed between lock release and broadcast — ` +
+                    `re-planned with updated funds: ${correction.ordersToPlace?.length || 0} creates, ` +
+                    `${correction.ordersToUpdate?.length || 0} updates`, 'info');
             }
             else {
-                manager.logger?.log?.(`[SPREAD] Fund state changed; no side has sufficient funds for re-plan. Skipping cycle.`, 'warn');
+                manager.logger?.log?.(`[SPREAD] Fund state changed; re-plan produced no viable orders. Skipping cycle.`, 'warn');
                 return { ordersPlaced: 0, partialsMoved: 0 };
             }
         }
@@ -1825,16 +1886,65 @@ async function prepareSpreadCorrectionOrders(manager, preferredSide) {
         edgePartial = partials[0];
         manager.logger?.log?.(`[SPREAD-CORRECTION] Identified partial order at ${edgePartial.price} for update`, 'debug');
     }
-    // Primary candidates: SPREAD-type slots adjacent to the gap.
+    // Boundary-correct type computation.  Used by both candidate pools below to ensure
+    // spread correction does not re-activate slots whose current boundary position
+    // places them on the wrong side or in the spread zone — doing so would compound
+    // inventory at prices where the bot already traded.
+    //
+    // The natural type of a slot is derived from its position in the price-sorted rail
+    // relative to boundaryIdx + gapSlots: indices in [0, boundaryIdx] are BUY, indices
+    // in [boundaryIdx + gapSlots + 1, N-1] are SELL, the middle band is SPREAD.
+    const allSlotsByPrice = allOrders
+        .filter((o) => o.price != null && Number.isFinite(o.price))
+        .sort((a, b) => a.price - b.price);
+    const slotIndexMap = new Map(allSlotsByPrice.map((o, i) => [o.id, i]));
+    const gapSlots = calculateGapSlots(manager.config?.incrementPercent, manager.config?.targetSpreadPercent, manager.config?.gridLimits);
+    // Sync boundary from current fund state to avoid stale boundaryIdx causing
+    // getSlotCorrectType to misclassify slots (e.g. after fills shifted the
+    // boundary but the COW commit hasn't updated it yet).  This is safe under
+    // the grid lock — no concurrent modifications can race with this read.
+    const boundarySync = (0, system_1.syncBoundaryToFunds)(manager);
+    if (boundarySync.changed && boundarySync.newIdx !== undefined) {
+        manager.boundaryIdx = boundarySync.newIdx;
+    }
+    const bIdx = manager.boundaryIdx ?? 0;
+    const buyEndIdx = bIdx;
+    const sellStartIdx = bIdx + Number(gapSlots) + 1;
+    const getSlotCorrectType = (slot) => {
+        const idx = slotIndexMap.get(slot.id);
+        if (idx === undefined)
+            return slot.type;
+        if (idx <= buyEndIdx)
+            return constants_1.ORDER_TYPES.BUY;
+        if (idx >= sellStartIdx)
+            return constants_1.ORDER_TYPES.SELL;
+        return constants_1.ORDER_TYPES.SPREAD;
+    };
+    // Primary candidates: SPREAD-type slots adjacent to the gap.  Filter by
+    // boundary-correct type so a SPREAD slot that, after a boundary shift, now sits
+    // in the BUY or SELL zone is excluded — it would otherwise be placed on the
+    // correction side at a price the grid already considers the opposite side.
     const typedSpreadCandidates = allOrders
-        .filter((o) => o.type === constants_1.ORDER_TYPES.SPREAD && (0, order_1.isSlotAvailable)(o))
+        .filter((o) => o.type === constants_1.ORDER_TYPES.SPREAD
+        && (0, order_1.isSlotAvailable)(o)
+        && getSlotCorrectType(o) === railType)
         .sort((a, b) => railType === constants_1.ORDER_TYPES.BUY ? a.price - b.price : b.price - a.price)
         .slice(0, missingSlots);
     // Secondary candidates: orphaned virtual slots of the correct side-type that have
     // lost their order (e.g. stale-cleaned after a race condition during a crash).
     // These sit inside the active window and are invisible to the SPREAD-type filter above.
+    //
+    // IMPORTANT: Filter by boundary-correct type so that filled-then-virtualized slots
+    // whose boundary position has moved into the spread or opposite zone are NOT
+    // re-activated on the stale side — doing so would compound inventory at prices
+    // where the bot already traded.  The boundary-correct type is computed from the
+    // current boundary index and the slot's price position in the sorted rail.
     const orphanedVirtualCandidates = allOrders
-        .filter((o) => o.type === railType && o.state === constants_1.ORDER_STATES.VIRTUAL && !o.orderId && Number(o.size || 0) === 0)
+        .filter((o) => o.type === railType
+        && o.state === constants_1.ORDER_STATES.VIRTUAL
+        && !o.orderId
+        && Number(o.size || 0) === 0
+        && getSlotCorrectType(o) === railType)
         .sort((a, b) => railType === constants_1.ORDER_TYPES.BUY ? b.price - a.price : a.price - b.price)
         .slice(0, missingSlots);
     // Merge: prefer orphaned virtuals (they already occupy correct grid positions) then