dexbot 1.4.7 → 1.4.9

package/dist/modules/order/accounting.js

@@ -111,9 +111,11 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 const constants_1 = require("../constants");
 const system_1 = require("./utils/system");
+const order_1 = require("./utils/order");
 const Format = __importStar(require("./format"));
 const fundRegistry = __importStar(require("../fund_registry"));
 const chainOrders = __importStar(require("../chain_orders"));
+const chain_orders_1 = require("../chain_orders");
 const math_1 = require("./utils/math");
 const processed_fill_store_1 = require("./processed_fill_store");
 const errors_1 = require("../utils/errors");
@@ -340,8 +342,20 @@ class Accountant {
         const mgr = this.manager;
         if (mgr._pauseFundRecalc > 0)
             return;
-        if (!mgr.funds)
-            this.resetFunds();
+        if (!mgr.funds) {
+            // Lazy funds init mutates mgr.funds wholesale, so run it under
+            // _fundLock when available (reentrant when the caller already
+            // holds it, e.g. manager.recalculateFunds). Guards the direct
+            // accountant.recalculateFunds() path against unlocked mutation.
+            if (mgr._fundLock) {
+                await mgr._fundLock.acquire(async () => {
+                    this.resetFunds();
+                });
+            }
+            else {
+                this.resetFunds();
+            }
+        }
         // Sync btsBalance from manager into funds for non-BTS pairs
         if (mgr.btsBalance) {
             mgr.funds.btsBalance = { ...mgr.funds.btsBalance, ...mgr.btsBalance };
@@ -365,8 +379,8 @@ class Accountant {
             // - SPREAD type: derive from price relation to startPrice (market midpoint)
             //   * price < startPrice → BUY side (lower prices are bids)
             //   * price >= startPrice → SELL side (higher prices are asks)
-            const isBuy = order.type === constants_1.ORDER_TYPES.BUY || (order.type === constants_1.ORDER_TYPES.SPREAD && order.price < mgr.config.startPrice);
-            const isSell = order.type === constants_1.ORDER_TYPES.SELL || (order.type === constants_1.ORDER_TYPES.SPREAD && order.price >= mgr.config.startPrice);
+            const isBuy = order.type === constants_1.ORDER_TYPES.BUY || (order.type === constants_1.ORDER_TYPES.SPREAD && (0, order_1.resolveSpreadOrderSide)(order.price, mgr.config.startPrice) === constants_1.ORDER_TYPES.BUY);
+            const isSell = order.type === constants_1.ORDER_TYPES.SELL || (order.type === constants_1.ORDER_TYPES.SPREAD && (0, order_1.resolveSpreadOrderSide)(order.price, mgr.config.startPrice) === constants_1.ORDER_TYPES.SELL);
             if (isBuy) {
                 if (isActive) {
                     gridBuy += size;
@@ -457,6 +471,19 @@ class Accountant {
      * @private
      */
     async _verifyFundInvariants(mgr, chainFreeBuy, chainFreeSell, chainBuy, chainSell, actualBuy, actualSell) {
+        // Half-baked guard: while a fill batch is mid-accounting, the balance
+        // snapshot may already reflect the just-filled orders on-chain while
+        // the grid still counts them as committed (or the reverse during
+        // optimistic fill adjustments). Running Total = Free + Committed in
+        // that window produces a spurious CRITICAL equal to the batch size.
+        // The check is deferred until the batch settles (grid mutation +
+        // authoritative re-anchor) and is re-run by the resume recalculation.
+        if ((mgr._fillBatchInFlight ?? 0) > 0) {
+            if (mgr.logger?.level === 'debug') {
+                mgr.logger.log('[INVARIANT] Fill batch in flight; deferring fund-invariant check until batch settles.', 'debug');
+            }
+            return;
+        }
         const buyPrecision = mgr.assets?.assetB?.precision;
         const sellPrecision = mgr.assets?.assetA?.precision;
         if (!Number.isFinite(buyPrecision) || !Number.isFinite(sellPrecision)) {
@@ -588,7 +615,25 @@ class Accountant {
         // proceeds, so the temporary invariant tolerance is no longer needed.
         mgr._orphanFillsCreditedAt = null;
         // 2. Sync from open orders
-        const openOrders = await chainOrders.readOpenOrders(accountRef);
+        // An empty/truncated read is ambiguous — the account is either genuinely
+        // empty or the node is lagging/capped (fresh orders omitted). Running
+        // syncFromOpenOrders on it would let pass-1 phantom cleanup virtualize
+        // ACTIVE/PARTIAL slots that are live on chain, after which the next
+        // cycle re-creates them as duplicates. Skip the sync and defer to the
+        // next reconcile cycle, mirroring the other recovery-read guards.
+        const openOrders = await (0, chain_orders_1.readOpenOrdersGuarded)(chainOrders, accountRef, {
+            log: (message, level) => mgr.logger?.log?.(message, level),
+            label: 'RECOVERY',
+            detail: 'during state recovery',
+            deferEmpty: true,
+        });
+        if (openOrders === null) {
+            return {
+                isValid: false,
+                deferred: true,
+                reason: 'Recovery sync skipped: ambiguous chain read (truncated or empty) - node may be lagging; deferring to next reconcile cycle',
+            };
+        }
         // Recovery runs after fetchAccountTotals() has refreshed authoritative balances
         // from chain. During this pass we only want to reconcile grid structure/order
         // mapping against open orders; re-applying optimistic accounting deltas here
@@ -692,6 +737,19 @@ class Accountant {
         mgr.logger?.log?.(`[RECOVERY] ${violationType} - attempting state recovery (attempt ${state.attemptCount}${hasAttemptLimit ? `/${maxAttemptsRaw}` : ''})...`, 'warn');
         try {
             const validation = await this._performStateRecovery(mgr);
+            if (validation?.deferred) {
+                // Ambiguous (empty/truncated) chain read: the recovery was
+                // DEFERRED, not failed. Roll back the attempt increment so an
+                // ambiguous snapshot can never burn the attempt budget toward
+                // MAX_RECOVERY_ATTEMPTS (which blocks all CREATEs until the
+                // next fill/sync cycle), and skip the structural resync — a
+                // full grid reset driven by an ambiguous read would
+                // virtualize live slots via pass-1 phantom cleanup.
+                state.attemptCount = Math.max(0, state.attemptCount - 1);
+                state.lastFailureAt = 0;
+                mgr.logger?.log?.(`[RECOVERY] State recovery deferred (${validation.reason}); attempt not counted`, 'warn');
+                return false;
+            }
             if (validation.isValid) {
                 mgr.logger?.log?.('[RECOVERY] State recovery succeeded', 'info');
                 state.structuralResyncRequested = false;
@@ -941,92 +999,142 @@ class Accountant {
         const mgr = this.manager;
         if (!oldOrder || !newOrder)
             return;
-        // Ensure a mutable copy: _resolveBtsFeeLifecycle mutates btsFeeState on
-        // newOrder (line 230), but the caller may pass a frozen master-grid order.
-        if (Object.isFrozen(newOrder)) {
-            newOrder = { ...newOrder };
-        }
+        // If this transition will LOCK capital, ensure a fresh accountTotals snapshot
+        // BEFORE acquiring _fundLock. The refresh is a live chain RPC (30s timeout,
+        // 3 retries, then node failover) — running it while holding _fundLock would
+        // stall every other fund-critical waiter (deductBtsFees, setAccountTotals,
+        // persistGrid) and, past the 30s acquire timeout, force them to throw instead
+        // of deferring. Only the plain mutation runs under the lock.
+        let preLockRefreshFailed = false;
         if (!skipAssetAccounting) {
             const oldIsActive = (oldOrder.state === constants_1.ORDER_STATES.ACTIVE || oldOrder.state === constants_1.ORDER_STATES.PARTIAL);
             const newIsActive = (newOrder.state === constants_1.ORDER_STATES.ACTIVE || newOrder.state === constants_1.ORDER_STATES.PARTIAL);
-            const oldSize = toFiniteNumber(oldOrder.size);
-            const newSize = toFiniteNumber(newOrder.size);
-            // 1. Handle Capital Commitment (Moves between FREE and LOCKED)
-            // For COMMITMENT: Use GRID state (isActive), not blockchain ID
-            const oldGridCommitted = oldIsActive ? oldSize : 0;
-            const newGridCommitted = newIsActive ? newSize : 0;
-            const commitmentDelta = newGridCommitted - oldGridCommitted;
-            const newSideType = this._resolveOrderSide(newOrder, oldOrder);
-            const oldSideType = this._resolveOrderSide(oldOrder, newOrder);
-            const sideForPrecision = newSideType || oldSideType;
-            if (mgr.logger && mgr.logger.level === 'debug') {
-                mgr.logger.log(`[ACCOUNTING] updateOptimisticFreeBalance: id=${newOrder.id}, type=${newOrder.type}, ` +
-                    `state=${oldOrder.state}->${newOrder.state}, ` +
-                    `size=${Format.formatSizeByOrderType(oldSize, sideForPrecision ?? '', mgr.assets)}->${Format.formatSizeByOrderType(newSize, sideForPrecision ?? '', mgr.assets)}, ` +
-                    `delta=${Format.formatSizeByOrderType(commitmentDelta, sideForPrecision ?? '', mgr.assets)}, context=${context}`, 'debug');
+            const willLock = (newIsActive ? toFiniteNumber(newOrder.size) : 0) - (oldIsActive ? toFiniteNumber(oldOrder.size) : 0) > 0;
+            if (willLock) {
+                try {
+                    const pre = await mgr.refreshAccountTotalsIfStale();
+                    preLockRefreshFailed = !pre.ok;
+                }
+                catch (err) {
+                    preLockRefreshFailed = true;
+                    mgr.logger?.log?.(`[ACCOUNTING] pre-lock accountTotals refresh error: ${(0, errors_1.getErrorMessage)(err)}`, 'warn');
+                }
             }
-            if (commitmentDelta > 0) {
-                // Lock capital: move from Free to Committed
-                const commitmentSide = newSideType || newOrder.type;
-                const result = await this.tryDeductFromChainFree(commitmentSide, commitmentDelta, `${context}`);
-                if (!result.ok) {
-                    const failure = {
-                        code: result.reason === 'stale' ? 'ACCOUNTING_STALE_ACCOUNT_TOTALS' : 'ACCOUNTING_COMMITMENT_FAILED',
-                        side: commitmentSide,
-                        amount: commitmentDelta,
-                        context,
-                        reason: result.reason,
-                        at: Date.now()
-                    };
-                    mgr._lastAccountingFailure = failure;
-                    if (result.reason === 'stale') {
-                        mgr.logger?.log?.(`[ACCOUNTING] Stale accountTotals: skipping optimistic lock of ${Format.formatAmount8(commitmentDelta)} for ${commitmentSide} during ${context}. Recovery scheduled.`, 'warn');
-                        // Schedule recovery but DON'T throw — staleness is transient.
-                        // The batch already confirmed on-chain; aborting the commit
-                        // would orphan a successful broadcast from the grid state.
-                    }
-                    else {
-                        mgr.logger?.log?.(`[ACCOUNTING] CRITICAL: Failed to lock ${Format.formatAmount8(commitmentDelta)} for ${commitmentSide} during ${context}. Scheduling recovery.`, 'error');
-                        if (mgr._throwOnIllegalState) {
-                            const err = new Error(`CRITICAL ACCOUNTING STATE: failed to lock ${Format.formatAmount8(commitmentDelta)} ${commitmentSide} during ${context}`);
-                            err.code = 'ACCOUNTING_COMMITMENT_FAILED';
-                            throw err;
+        }
+        return await mgr._fundLock.acquire(async () => {
+            // Ensure a copy: _resolveBtsFeeLifecycle mutates btsFeeState on
+            // newOrder (line 230), but the caller may pass a frozen master-grid order.
+            if (Object.isFrozen(newOrder)) {
+                newOrder = { ...newOrder };
+            }
+            if (!skipAssetAccounting) {
+                const oldIsActive = (oldOrder.state === constants_1.ORDER_STATES.ACTIVE || oldOrder.state === constants_1.ORDER_STATES.PARTIAL);
+                const newIsActive = (newOrder.state === constants_1.ORDER_STATES.ACTIVE || newOrder.state === constants_1.ORDER_STATES.PARTIAL);
+                const oldSize = toFiniteNumber(oldOrder.size);
+                const newSize = toFiniteNumber(newOrder.size);
+                // 1. Handle Capital Commitment (Moves between FREE and LOCKED)
+                // For COMMITMENT: Use GRID state (isActive), not blockchain ID
+                const oldGridCommitted = oldIsActive ? oldSize : 0;
+                const newGridCommitted = newIsActive ? newSize : 0;
+                const commitmentDelta = newGridCommitted - oldGridCommitted;
+                const newSideType = this._resolveOrderSide(newOrder, oldOrder);
+                const oldSideType = this._resolveOrderSide(oldOrder, newOrder);
+                const sideForPrecision = newSideType || oldSideType;
+                if (mgr.logger && mgr.logger.level === 'debug') {
+                    mgr.logger.log(`[ACCOUNTING] updateOptimisticFreeBalance: id=${newOrder.id}, type=${newOrder.type}, ` +
+                        `state=${oldOrder.state}->${newOrder.state}, ` +
+                        `size=${Format.formatSizeByOrderType(oldSize, sideForPrecision ?? '', mgr.assets)}->${Format.formatSizeByOrderType(newSize, sideForPrecision ?? '', mgr.assets)}, ` +
+                        `delta=${Format.formatSizeByOrderType(commitmentDelta, sideForPrecision ?? '', mgr.assets)}, context=${context}`, 'debug');
+                }
+                if (commitmentDelta > 0) {
+                    // Lock capital: move from Free to Committed
+                    const commitmentSide = newSideType || newOrder.type;
+                    let result = await this.tryDeductFromChainFree(commitmentSide, commitmentDelta, `${context}`);
+                    // Fix staleness in the first place: a stale snapshot refused
+                    // the lock. When the pre-lock refresh was skipped (not needed)
+                    // or is in flight and this deduction still races a fresh window,
+                    // refresh accountTotals from chain and retry once. But if the
+                    // pre-lock refresh already FAILED, the chain is unhealthy — do
+                    // NOT re-run the 30s/3-retry/node-failover RPC while holding
+                    // _fundLock (that would stall every other fund-critical waiter
+                    // exactly like the regression this rework eliminated). Fall
+                    // through to the failure/recovery path below instead.
+                    if (!result.ok && result.reason === 'stale' && !preLockRefreshFailed) {
+                        mgr.logger?.log?.(`[ACCOUNTING] Stale accountTotals during ${context}; refreshing from chain before retrying optimistic lock.`, 'warn');
+                        const refresh = await mgr.refreshAccountTotalsIfStale();
+                        if (refresh.ok) {
+                            result = await this.tryDeductFromChainFree(commitmentSide, commitmentDelta, `${context} (post-refresh retry)`);
+                        }
+                        else {
+                            mgr.logger?.log?.(`[ACCOUNTING] accountTotals refresh failed (${refresh.reason}); cannot retry optimistic lock of ${Format.formatAmount8(commitmentDelta)} for ${commitmentSide}.`, 'warn');
                         }
                     }
-                    // Promote recovery from fire-and-forget to a stored promise.
-                    // Subsequent callers can await the in-flight recovery before
-                    // attempting new deductions, reducing redundant recovery cycles.
-                    if (!mgr._pendingRecovery) {
-                        mgr._pendingRecovery = this._attemptFundRecovery(mgr, 'Optimistic commitment deduction failure')
-                            .catch((err) => {
-                            mgr.logger?.log?.(`[RECOVERY] Immediate recovery scheduling failed: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
-                            mgr._recoveryState = { ...mgr._recoveryState, lastFailureAt: Date.now() };
-                        })
-                            .finally(() => {
-                            mgr._pendingRecovery = null;
-                        });
+                    else if (!result.ok && result.reason === 'stale' && preLockRefreshFailed) {
+                        // The pre-lock refresh already failed (chain unhealthy) so we
+                        // refuse to re-run the blocking RPC under _fundLock. Fall
+                        // through to the stale failure/recovery path below; the batch
+                        // stays accounted for and recovery is scheduled.
+                        mgr.logger?.log?.(`[ACCOUNTING] Stale accountTotals during ${context}; pre-lock refresh failed, skipping in-lock refresh to avoid holding _fundLock across a blocking chain RPC.`, 'warn');
                     }
-                    else {
-                        mgr.logger?.log?.(`[ACCOUNTING] Recovery already in-flight; waiting for existing recovery to complete before retry deduction of ${Format.formatAmount8(commitmentDelta)} for ${commitmentSide}`, 'warn');
+                    if (!result.ok) {
+                        const failure = {
+                            code: result.reason === 'stale' ? 'ACCOUNTING_STALE_ACCOUNT_TOTALS' : 'ACCOUNTING_COMMITMENT_FAILED',
+                            side: commitmentSide,
+                            amount: commitmentDelta,
+                            context,
+                            reason: result.reason,
+                            at: Date.now()
+                        };
+                        mgr._lastAccountingFailure = failure;
+                        if (result.reason === 'stale') {
+                            mgr.logger?.log?.(`[ACCOUNTING] Stale accountTotals: skipping optimistic lock of ${Format.formatAmount8(commitmentDelta)} for ${commitmentSide} during ${context}. Recovery scheduled.`, 'warn');
+                            // Schedule recovery but DON'T throw — staleness is transient.
+                            // The batch already confirmed on-chain; aborting the commit
+                            // would orphan a successful broadcast from the grid state.
+                        }
+                        else {
+                            mgr.logger?.log?.(`[ACCOUNTING] CRITICAL: Failed to lock ${Format.formatAmount8(commitmentDelta)} for ${commitmentSide} during ${context}. Scheduling recovery.`, 'error');
+                            if (mgr._throwOnIllegalState) {
+                                const err = new Error(`CRITICAL ACCOUNTING STATE: failed to lock ${Format.formatAmount8(commitmentDelta)} ${commitmentSide} during ${context}`);
+                                err.code = 'ACCOUNTING_COMMITMENT_FAILED';
+                                throw err;
+                            }
+                        }
+                        // Promote recovery from fire-and-forget to a stored promise.
+                        // Subsequent callers can await the in-flight recovery before
+                        // attempting new deductions, reducing redundant recovery cycles.
+                        if (!mgr._pendingRecovery) {
+                            mgr._pendingRecovery = this._attemptFundRecovery(mgr, 'Optimistic commitment deduction failure')
+                                .catch((err) => {
+                                mgr.logger?.log?.(`[RECOVERY] Immediate recovery scheduling failed: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
+                                mgr._recoveryState = { ...mgr._recoveryState, lastFailureAt: Date.now() };
+                            })
+                                .finally(() => {
+                                mgr._pendingRecovery = null;
+                            });
+                        }
+                        else {
+                            mgr.logger?.log?.(`[ACCOUNTING] Recovery already in-flight; waiting for existing recovery to complete before retry deduction of ${Format.formatAmount8(commitmentDelta)} for ${commitmentSide}`, 'warn');
+                        }
                     }
                 }
+                else if (commitmentDelta < 0) {
+                    // Release capital: move from Committed back to Free
+                    const releaseSide = oldSideType || oldOrder.type;
+                    await this.addToChainFree(releaseSide, Math.abs(commitmentDelta), `${context}`);
+                }
             }
-            else if (commitmentDelta < 0) {
-                // Release capital: move from Committed back to Free
-                const releaseSide = oldSideType || oldOrder.type;
-                await this.addToChainFree(releaseSide, Math.abs(commitmentDelta), `${context}`);
-            }
-        }
-        // 2. Handle BTS blockchain fee lifecycle.
-        // BitShares stores create/update fees as order.deferred_fee and later
-        // refunds or charges that deferred fee on fill, update, or cancel.
-        const btsOrderType = this._getBtsOrderType();
-        if (btsOrderType) {
-            const { balanceDelta } = this._resolveBtsFeeLifecycle(oldOrder, newOrder, context, fee);
-            if (Math.abs(balanceDelta) > 0) {
-                await this.adjustTotalBalance(btsOrderType, balanceDelta, `${context}-fee`);
+            // 2. Handle BTS blockchain fee lifecycle.
+            // BitShares stores create/update fees as order.deferred_fee and later
+            // refunds or charges that deferred fee on fill, update, or cancel.
+            const btsOrderType = this._getBtsOrderType();
+            if (btsOrderType) {
+                const { balanceDelta } = this._resolveBtsFeeLifecycle(oldOrder, newOrder, context, fee);
+                if (Math.abs(balanceDelta) > 0) {
+                    await this.adjustTotalBalance(btsOrderType, balanceDelta, `${context}-fee`);
+                }
             }
-        }
+        });
     }
     /**
      * Deduct BTS fees using adjustTotalBalance.
@@ -1040,10 +1148,9 @@ class Accountant {
      */
     async deductBtsFees(requestedSide = null) {
         const mgr = this.manager;
-        // Early returns for no work needed
-        if (!mgr.funds || !mgr.funds.btsFeesOwed || mgr.funds.btsFeesOwed <= 0)
-            return;
-        if (!mgr.accountTotals)
+        // Early returns for no work needed (existence checks only — value reads
+        // happen inside _fundLock below so overlapping calls cannot double-deduct)
+        if (!mgr.funds || !mgr.accountTotals)
             return;
         const btsSide = (0, math_1.getBtsSide)(mgr.config?.assetA, mgr.config?.assetB);
         const normalizedRequestedSide = (requestedSide === 'buy' || requestedSide === 'sell') ? requestedSide : null;
@@ -1054,26 +1161,37 @@ class Accountant {
         }
         if (!side)
             return;
-        const fees = mgr.funds.btsFeesOwed;
-        const orderType = (side === 'buy') ? constants_1.ORDER_TYPES.BUY : constants_1.ORDER_TYPES.SELL;
-        const freeKey = (side === 'buy') ? 'buyFree' : 'sellFree';
-        const chainFree = mgr.accountTotals[freeKey] || 0;
-        // SUFFICIENCY CHECK: Defer if insufficient funds
-        if (chainFree < fees) {
+        let settled = false;
+        // ATOMIC: read btsFeesOwed + sufficiency check + deduction + reset in one
+        // lock. Two overlapping calls then serialize: the second sees fees=0 and
+        // no-ops instead of double-deducting a stale amount.
+        await mgr._fundLock.acquire(async () => {
+            const fees = mgr.funds.btsFeesOwed;
+            if (!fees || fees <= 0)
+                return;
+            const orderType = (side === 'buy') ? constants_1.ORDER_TYPES.BUY : constants_1.ORDER_TYPES.SELL;
+            const freeKey = (side === 'buy') ? 'buyFree' : 'sellFree';
+            const chainFree = mgr.accountTotals[freeKey] || 0;
+            // SUFFICIENCY CHECK: Defer if insufficient funds
+            if (chainFree < fees) {
+                if (mgr.logger && mgr.logger.level === 'debug') {
+                    mgr.logger.log(`[BTS-FEE] Deferring settlement: need ${Format.formatAmount8(fees)}, have ${Format.formatAmount8(chainFree)}`, 'debug');
+                }
+                return;
+            }
+            this._adjustTotalBalanceLocked(orderType, -fees, 'bts-fee-settlement');
+            mgr.funds.btsFeesOwed = 0;
+            settled = true;
             if (mgr.logger && mgr.logger.level === 'debug') {
-                mgr.logger.log(`[BTS-FEE] Deferring settlement: need ${Format.formatAmount8(fees)}, have ${Format.formatAmount8(chainFree)}`, 'debug');
+                mgr.logger.log(`[BTS-FEE] Settled: ${Format.formatAmount8(fees)} BTS`, 'debug');
             }
-            return;
-        }
-        // FULL DEDUCTION from chainFree
-        await this.adjustTotalBalance(orderType, -fees, 'bts-fee-settlement');
-        if (mgr.logger && mgr.logger.level === 'debug') {
-            mgr.logger.log(`[BTS-FEE] Settled: ${Format.formatAmount8(fees)} BTS`, 'debug');
+        });
+        // Recalculate funds only when something was actually settled — the
+        // no-op calls (no fees owed, or deferred for insufficient funds) skip
+        // the full grid iteration.
+        if (settled) {
+            await mgr.recalculateFunds();
         }
-        // Reset fees after successful settlement
-        mgr.funds.btsFeesOwed = 0;
-        // Recalculate funds to update all tracking metrics
-        await mgr.recalculateFunds();
     }
     /**
      * Validate that a proposed Target Grid fits within the account's total available funds.