dexbot 1.3.2 → 1.4.0

package/dist/modules/dexbot_fill_runtime.js

@@ -1,8 +1,26 @@
 "use strict";
 /** Fill processing runtime - handles order fill events and replay-safe accounting */
-const { buildFillKey } = require('./order/utils/order');
-const { PROCESSED_FILL_PERSISTENCE_MODES } = require('./order/processed_fill_store');
-const { NATIVE_CLIENT } = require('./constants');
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wireProcessedFillTracking = wireProcessedFillTracking;
+exports.flushProcessedFillPersistence = flushProcessedFillPersistence;
+exports.flushProcessedFillPersistenceForKeys = flushProcessedFillPersistenceForKeys;
+exports.discardPendingProcessedFillPersistence = discardPendingProcessedFillPersistence;
+exports.buildOrphanFillFallbackKey = buildOrphanFillFallbackKey;
+exports.applyReplaySafeFillAccounting = applyReplaySafeFillAccounting;
+exports.applyReplaySafeTrackedFillAccounting = applyReplaySafeTrackedFillAccounting;
+exports.applyReplaySafeOrphanFillAccounting = applyReplaySafeOrphanFillAccounting;
+exports.createFillCallback = createFillCallback;
+exports.maxConsecutiveFillConsumerFailures = maxConsecutiveFillConsumerFailures;
+exports.computeFillConsumerBackoffMs = computeFillConsumerBackoffMs;
+exports.scheduleFillConsumerRestart = scheduleFillConsumerRestart;
+exports.consumeFillQueue = consumeFillQueue;
+exports.processFillsWithBootstrapMode = processFillsWithBootstrapMode;
+const processed_fill_store_1 = require("./order/processed_fill_store");
+const constants_1 = require("./constants");
+const errors_1 = require("./utils/errors");
+function buildFillKey(...args) { return require('./order/utils/order').buildFillKey(...args); }
+function correctAllPriceMismatches(...args) { return require('./order/utils/order').correctAllPriceMismatches(...args); }
+function retryPersistenceIfNeeded(...args) { return require('./order/utils/system').retryPersistenceIfNeeded(...args); }
 /**
  * Wire processed fill tracking into the manager and processed fill store.
  * Establishes the bidirectional link between the runtime's processed fill store
@@ -99,7 +117,7 @@ function buildOrphanFillFallbackKey(bot, fill) {
  * @param {boolean} [options.allowOrphanFallbackKey=false] - Allow degraded orphan fallback key
  * @returns {Promise<import('./types').ReplaySafeFillResult>}
  */
-async function applyReplaySafeFillAccounting(bot, fill, fillOp, { missingKeyMessage, fallbackKeyMessage, replayMessage, errorMessage, logger = bot.manager?.logger, missingKeyLevel = 'warn', fallbackKeyLevel = 'warn', replayLevel = 'debug', persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.IMMEDIATE, allowOrphanFallbackKey = false } = {}) {
+async function applyReplaySafeFillAccounting(bot, fill, fillOp, { missingKeyMessage, fallbackKeyMessage, replayMessage, errorMessage, logger = bot.manager?.logger, missingKeyLevel = 'warn', fallbackKeyLevel = 'warn', replayLevel = 'debug', persistenceMode = processed_fill_store_1.PROCESSED_FILL_PERSISTENCE_MODES.IMMEDIATE, allowOrphanFallbackKey = false } = {}) {
     let fillKey = buildFillKey(fill);
     let usedFallbackKey = false;
     if (!fillKey && allowOrphanFallbackKey) {
@@ -146,12 +164,12 @@ async function applyReplaySafeFillAccounting(bot, fill, fillOp, { missingKeyMess
  * @param {string} [options.persistenceMode='batched'] - Processed fill persistence mode
  * @returns {Promise<import('./types').ReplaySafeFillResult>}
  */
-async function applyReplaySafeTrackedFillAccounting(bot, fill, fillOp, { context, logger = bot.manager?.logger, replayMessage, persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.BATCHED } = {}) {
+async function applyReplaySafeTrackedFillAccounting(bot, fill, fillOp, { context, logger = bot.manager?.logger, replayMessage, persistenceMode = processed_fill_store_1.PROCESSED_FILL_PERSISTENCE_MODES.BATCHED } = {}) {
     return applyReplaySafeFillAccounting(bot, fill, fillOp, {
         logger,
         missingKeyMessage: (op) => `[${context}] Missing fill history id for ${op.order_id}; deferring to open-orders sync`,
         replayMessage,
-        errorMessage: (op, _fill, err) => `[${context}] Failed to process accounting for ${op.order_id}: ${err.message}`,
+        errorMessage: (op, _fill, err) => `[${context}] Failed to process accounting for ${op.order_id}: ${(0, errors_1.getErrorMessage)(err)}`,
         persistenceMode
     });
 }
@@ -168,13 +186,13 @@ async function applyReplaySafeTrackedFillAccounting(bot, fill, fillOp, { context
  * @param {string} [options.persistenceMode='batched'] - Processed fill persistence mode
  * @returns {Promise<import('./types').ReplaySafeFillResult>}
  */
-async function applyReplaySafeOrphanFillAccounting(bot, fill, fillOp, { context, logger = bot.manager?.logger, replayMessage, persistenceMode = PROCESSED_FILL_PERSISTENCE_MODES.BATCHED } = {}) {
+async function applyReplaySafeOrphanFillAccounting(bot, fill, fillOp, { context, logger = bot.manager?.logger, replayMessage, persistenceMode = processed_fill_store_1.PROCESSED_FILL_PERSISTENCE_MODES.BATCHED } = {}) {
     return applyReplaySafeFillAccounting(bot, fill, fillOp, {
         logger,
         missingKeyMessage: (op) => `[${context}] Missing fill history id and orphan fallback key for ${op.order_id}; deferring to open-orders sync`,
         fallbackKeyMessage: (op) => `[${context}] Missing fill history id for orphan fill ${op.order_id}; using degraded orphan replay key for proceeds-only accounting`,
         replayMessage,
-        errorMessage: (op, _fill, err) => `[${context}] Failed to process accounting for ${op.order_id}: ${err.message}`,
+        errorMessage: (op, _fill, err) => `[${context}] Failed to process accounting for ${op.order_id}: ${(0, errors_1.getErrorMessage)(err)}`,
         persistenceMode,
         allowOrphanFallbackKey: true
     });
@@ -192,7 +210,7 @@ function createFillCallback(bot, chainOrders) {
             return;
         }
         if (bot.manager && !bot.config.dryRun && Array.isArray(fills) && fills.length > 0) {
-            const maxQueueDepth = NATIVE_CLIENT.SUBSCRIPTIONS.MAX_INCOMING_FILL_QUEUE;
+            const maxQueueDepth = constants_1.NATIVE_CLIENT.SUBSCRIPTIONS.MAX_INCOMING_FILL_QUEUE;
             if (bot._incomingFillQueue.length + fills.length > maxQueueDepth) {
                 const message = `Incoming fill queue back-pressure: ${bot._incomingFillQueue.length} queued + ${fills.length} incoming exceeds limit ${maxQueueDepth}`;
                 bot._warn(message);
@@ -200,21 +218,546 @@ function createFillCallback(bot, chainOrders) {
             }
             bot._markGridActivity?.('fill queued');
             bot._incomingFillQueue.push(...fills);
-            bot._consumeFillQueue(chainOrders).catch(err => {
-                bot._warn(`Fill queue consume failed: ${err.message}`);
+            bot._consumeFillQueue(chainOrders).catch((err) => {
+                bot._warn(`Fill queue consume failed: ${(0, errors_1.getErrorMessage)(err)}`);
             });
         }
     };
 }
-module.exports = {
-    wireProcessedFillTracking,
-    flushProcessedFillPersistence,
-    flushProcessedFillPersistenceForKeys,
-    discardPendingProcessedFillPersistence,
-    buildOrphanFillFallbackKey,
-    applyReplaySafeFillAccounting,
-    applyReplaySafeTrackedFillAccounting,
-    applyReplaySafeOrphanFillAccounting,
-    createFillCallback
-};
+/**
+ * Returns the maximum consecutive fill consumer failures allowed before backoff.
+ * @param {import('./dexbot_class').DEXBot} bot
+ * @returns {number}
+ */
+function maxConsecutiveFillConsumerFailures(bot) {
+    return bot.config.fillProcessing?.MAX_CONSECUTIVE_CONSUMER_FAILURES ?? constants_1.FILL_PROCESSING.MAX_CONSECUTIVE_CONSUMER_FAILURES;
+}
+/**
+ * Compute the backoff delay for fill-consumer retries after the failure
+ * budget (MAX_CONSECUTIVE_CONSUMER_FAILURES) is exhausted. Each retry
+ * doubles the previous delay, capped at CONSUMER_BACKOFF_MAX_MS. The
+ * consumer NEVER permanently stops re-scheduling — it just slows down.
+ * @param {import('./dexbot_class').DEXBot} bot
+ * @param {number} failures The current consecutive-failure count.
+ * @returns {number} Delay in milliseconds before the next retry.
+ */
+function computeFillConsumerBackoffMs(bot, failures) {
+    const fp = bot.config.fillProcessing || constants_1.FILL_PROCESSING;
+    const initial = fp.CONSUMER_BACKOFF_INITIAL_MS;
+    const max = fp.CONSUMER_BACKOFF_MAX_MS;
+    const stepAfterMax = Math.max(0, failures - maxConsecutiveFillConsumerFailures(bot));
+    return Math.min(max, initial * Math.pow(2, stepAfterMax));
+}
+/**
+ * Schedule a fill consumer restart with exponential backoff when the
+ * failure budget is exhausted, or immediate retry via setImmediate when
+ * within the budget. The consumer NEVER permanently stops re-scheduling.
+ * @param {import('./dexbot_class').DEXBot} bot
+ * @param {Object} chainOrders - Chain orders module
+ */
+function scheduleFillConsumerRestart(bot, chainOrders) {
+    const failures = bot._consecutiveConsumeFailures;
+    if (failures >= maxConsecutiveFillConsumerFailures(bot)) {
+        const backoffMs = computeFillConsumerBackoffMs(bot, failures);
+        const elapsedSec = bot._consumeFailureFirstAt
+            ? Math.round((Date.now() - bot._consumeFailureFirstAt) / constants_1.TIMING.MILLISECONDS_PER_SECOND)
+            : null;
+        const elapsed = elapsedSec !== null ? `${elapsedSec}s` : 'unknown';
+        const sustainedLevel = (failures >= 20 || (elapsedSec !== null && elapsedSec >= 900))
+            ? 'critical'
+            : (failures >= 10 || (elapsedSec !== null && elapsedSec >= 300))
+                ? 'error'
+                : 'warn';
+        bot._log(`[FILL-QUEUE] Fill consumer has failed ${failures} consecutive times over ${elapsed}; ` +
+            `backing off ${Math.round(backoffMs / constants_1.TIMING.MILLISECONDS_PER_SECOND)}s before retry. ` +
+            `Queue: ${bot._incomingFillQueue.length} fills.`, sustainedLevel);
+        setTimeout(() => {
+            if (bot._shuttingDown)
+                return;
+            bot._consumeFillQueue(chainOrders).catch((err) => {
+                if (!bot._consumeFailureFirstAt) {
+                    bot._consumeFailureFirstAt = Date.now();
+                }
+                bot._consecutiveConsumeFailures++;
+                const newFailures = bot._consecutiveConsumeFailures;
+                const newElapsedSec = bot._consumeFailureFirstAt
+                    ? Math.round((Date.now() - bot._consumeFailureFirstAt) / constants_1.TIMING.MILLISECONDS_PER_SECOND)
+                    : null;
+                const resumeLevel = (newFailures >= 20 || (newElapsedSec !== null && newElapsedSec >= 900))
+                    ? 'critical'
+                    : (newFailures >= 10 || (newElapsedSec !== null && newElapsedSec >= 300))
+                        ? 'error'
+                        : 'warn';
+                bot._log(`Fill consumer resume after backoff failed ` +
+                    `(${newFailures} total, ` +
+                    `next backoff ${Math.round(computeFillConsumerBackoffMs(bot, newFailures) / constants_1.TIMING.MILLISECONDS_PER_SECOND)}s): ` +
+                    `${(0, errors_1.getErrorMessage)(err)}`, resumeLevel);
+                bot._scheduleFillConsumerRestart(chainOrders);
+            });
+        }, backoffMs);
+        return;
+    }
+    setImmediate(() => bot._consumeFillQueue(chainOrders).catch((err) => {
+        if (!bot._consumeFailureFirstAt) {
+            bot._consumeFailureFirstAt = Date.now();
+        }
+        bot._consecutiveConsumeFailures++;
+        const remaining = maxConsecutiveFillConsumerFailures(bot) - bot._consecutiveConsumeFailures;
+        bot._log(`Fill consumer failed (${bot._consecutiveConsumeFailures}/${maxConsecutiveFillConsumerFailures(bot)}, ` +
+            `${remaining} attempts remaining): ${(0, errors_1.getErrorMessage)(err)}`, bot._consecutiveConsumeFailures >= 3 ? 'warn' : 'error');
+    }));
+}
+/**
+ * Process fills during bootstrap phase using the standard fill pipeline.
+ * Delegates to the same fill pipeline as the post-reset path.
+ * @param {import('./dexbot_class').DEXBot} bot
+ * @param {Object} chainOrders - Chain orders module for blockchain operations
+ * @returns {Promise<void>}
+ */
+async function processFillsWithBootstrapMode(bot, chainOrders) {
+    if (bot._incomingFillQueue.length === 0)
+        return;
+    const startTime = Date.now();
+    const fills = bot._incomingFillQueue.splice(0);
+    const validFills = [];
+    const processedFillKeys = new Set();
+    let requiresOpenOrdersSync = false;
+    for (const fill of fills) {
+        if (!fill || fill.op?.[0] !== 4)
+            continue;
+        const fillOp = fill.op[1];
+        const gridOrder = bot.manager.orders.get(fillOp.order_id) ||
+            Array.from(bot.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
+        if (!gridOrder) {
+            let orphanFillKey = buildFillKey(fill);
+            if (!orphanFillKey) {
+                orphanFillKey = bot._buildOrphanFillFallbackKey(fill);
+            }
+            if (orphanFillKey && !bot._isNewFillKey(orphanFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
+                continue;
+            }
+            bot.manager.logger.log(`[BOOTSTRAP] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
+            const accountingResult = await bot._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
+                context: 'BOOTSTRAP'
+            });
+            if (accountingResult.status === 'missing_key') {
+                requiresOpenOrdersSync = true;
+            }
+            continue;
+        }
+        const trackedFillKey = buildFillKey(fill);
+        if (trackedFillKey && !bot._isNewFillKey(trackedFillKey, processedFillKeys, '[BOOTSTRAP]', fillOp.order_id)) {
+            continue;
+        }
+        bot.manager.lockOrders([gridOrder.id]);
+        try {
+            const accountingResult = await bot._applyReplaySafeTrackedFillAccounting(fill, fillOp, {
+                context: 'BOOTSTRAP',
+                replayMessage: (op) => `[BOOTSTRAP] Replay detected for ${op.order_id}; skipping duplicate bootstrap rebalance`
+            });
+            if (accountingResult.status === 'missing_key') {
+                requiresOpenOrdersSync = true;
+                continue;
+            }
+            if (accountingResult.status !== 'applied') {
+                continue;
+            }
+            validFills.push({ ...fill, gridOrder });
+            const fillType = gridOrder.type === constants_1.ORDER_TYPES.BUY ? 'BUY' : 'SELL';
+            bot._log(`[BOOTSTRAP] Fill detected: ${fillType} order (${fillOp.is_maker !== false ? 'maker' : 'taker'})`);
+        }
+        finally {
+            bot.manager.unlockOrders([gridOrder.id]);
+        }
+    }
+    if (requiresOpenOrdersSync) {
+        bot._log('[BOOTSTRAP] Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
+        const bootstrapChainOpenOrders = await chainOrders.readOpenOrders(bot.accountId);
+        const syncResult = await bot.manager.syncFromOpenOrders(bootstrapChainOpenOrders);
+        if (syncResult.filledOrders?.length > 0) {
+            const queuedOrderIds = new Set(validFills.map((fill) => fill?.gridOrder?.orderId).filter(Boolean));
+            for (const filledOrder of syncResult.filledOrders) {
+                if (!filledOrder?.orderId || queuedOrderIds.has(filledOrder.orderId))
+                    continue;
+                validFills.push({ gridOrder: filledOrder });
+                queuedOrderIds.add(filledOrder.orderId);
+            }
+        }
+    }
+    await bot._flushProcessedFillPersistence('bootstrap-batch');
+    if (validFills.length === 0)
+        return;
+    try {
+        bot._log(`[BOOTSTRAP] Processing ${validFills.length} fill(s) through standard pipeline`, 'info');
+        const filledOrders = validFills.map((f) => f.gridOrder);
+        const result = await bot._processFillsWithBatching(filledOrders, new Set(), '[BOOTSTRAP] fill processing');
+        if (result.aborted) {
+            bot._warn('[BOOTSTRAP] Aborted batch due to illegal state; skipping grid persistence this cycle');
+        }
+        bot._metrics.fillsProcessed += validFills.length;
+        bot._metrics.fillProcessingTimeMs += Date.now() - startTime;
+    }
+    catch (err) {
+        bot._warn(`[BOOTSTRAP] Error processing fills: ${(0, errors_1.getErrorMessage)(err)}`);
+        bot.manager.logger.log(`[BOOTSTRAP] Fill error: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
+    }
+}
+/**
+ * Consume queued fills from incomingFillQueue and rebalance.
+ * Deduplicates fills against already-processed set (replay-safe), syncs filled
+ * orders from history or open orders mode, handles price mismatches, processes
+ * fills sequentially with interruptible rebalancing, and periodically cleans old
+ * fill records.
+ * @param {import('./dexbot_class').DEXBot} bot
+ * @param {Object} chainOrders - Chain orders module for blockchain operations
+ */
+async function consumeFillQueue(bot, chainOrders) {
+    const resetFailureWatchdogIfSet = () => {
+        if (bot._consecutiveConsumeFailures > 0 || bot._consumeFailureFirstAt > 0) {
+            bot._consecutiveConsumeFailures = 0;
+            bot._consumeFailureFirstAt = 0;
+        }
+    };
+    if (bot._incomingFillQueue.length === 0) {
+        resetFailureWatchdogIfSet();
+        return;
+    }
+    if (bot._shuttingDown) {
+        bot._warn('Fill processing skipped: shutdown in progress');
+        resetFailureWatchdogIfSet();
+        return;
+    }
+    if (bot._batchInFlight || bot._recoverySyncInFlight) {
+        bot.manager?.logger?.log?.(`Fill processing deferred: order pipeline active (${bot._incomingFillQueue.length} queued)`, 'debug');
+        resetFailureWatchdogIfSet();
+        return;
+    }
+    let pendingFillKeysForCurrentCycle = new Set();
+    try {
+        if (bot.manager.isBootstrapping()) {
+            let bootstrapSkipped = false;
+            await bot.manager._fillProcessingLock.acquire(async () => {
+                if (!bot.manager.isBootstrapping()) {
+                    bootstrapSkipped = true;
+                    return;
+                }
+                await bot._processFillsWithBootstrapMode(chainOrders);
+            });
+            if (bootstrapSkipped) {
+                resetFailureWatchdogIfSet();
+            }
+            return;
+        }
+        if (bot.manager._fillProcessingLock.getQueueLength() > 0) {
+            bot._metrics.lockContentionEvents++;
+            resetFailureWatchdogIfSet();
+            return;
+        }
+        await bot.manager._fillProcessingLock.acquire(async () => {
+            bot.manager._orphanFillsCreditedAt = null;
+            while (bot._incomingFillQueue.length > 0) {
+                const batchStartTime = Date.now();
+                bot._metrics.maxQueueDepth = Math.max(bot._metrics.maxQueueDepth, bot._incomingFillQueue.length);
+                const allFills = bot._incomingFillQueue.splice(0);
+                const validFills = [];
+                const processedFillKeys = new Set();
+                pendingFillKeysForCurrentCycle = new Set();
+                let requiresOpenOrdersSync = false;
+                for (const fill of allFills) {
+                    if (fill && fill.op && fill.op[0] === constants_1.FILL_PROCESSING.OPERATION_TYPE) {
+                        const fillOp = fill.op[1];
+                        const hasFillEconomics = fillOp?.pays?.asset_id && fillOp?.pays?.amount != null
+                            && fillOp?.receives?.asset_id && fillOp?.receives?.amount != null;
+                        if (chainOrders && typeof chainOrders.wasRecentlyOwnCancelled === 'function'
+                            && chainOrders.wasRecentlyOwnCancelled(fillOp.order_id)
+                            && !hasFillEconomics) {
+                            bot.manager.logger.log(`[SELF-CANCEL] Skipping non-economic fill artifact for order ${fillOp.order_id} (just cancelled by this bot)`, 'debug');
+                            continue;
+                        }
+                        const gridOrder = bot.manager.orders.get(fillOp.order_id) ||
+                            Array.from(bot.manager.orders.values()).find((o) => o.orderId === fillOp.order_id);
+                        if (!gridOrder) {
+                            const staleMarkedAt = bot._staleCleanedOrderIds.get(fillOp.order_id);
+                            if (staleMarkedAt != null) {
+                                const staleAgeMs = Date.now() - staleMarkedAt;
+                                if (staleAgeMs <= bot._staleCleanupRetentionMs) {
+                                    bot.manager.logger.log(`[ORPHAN-FILL] Skipping double-credit for stale-cleaned order ${fillOp.order_id} ` +
+                                        `(funds already freed by batch cleanup, age=${staleAgeMs}ms)`, 'warn');
+                                    continue;
+                                }
+                                bot._staleCleanedOrderIds.delete(fillOp.order_id);
+                            }
+                            let orphanFillKey = buildFillKey(fill);
+                            if (!orphanFillKey) {
+                                orphanFillKey = bot._buildOrphanFillFallbackKey(fill);
+                            }
+                            if (orphanFillKey && !bot._isNewFillKey(orphanFillKey, processedFillKeys, '[ORPHAN-FILL]', fillOp.order_id)) {
+                                continue;
+                            }
+                            bot.manager.logger.log(`[ORPHAN-FILL] Processing funds for unknown order ${fillOp.order_id} (not in grid but crediting proceeds)`, 'warn');
+                            const accountingResult = await bot._applyReplaySafeOrphanFillAccounting(fill, fillOp, {
+                                context: 'ORPHAN-FILL',
+                                replayMessage: (op) => `[ORPHAN-FILL] Replay detected for ${op.order_id}; skipping duplicate credit`
+                            });
+                            if (accountingResult.status === 'missing_key') {
+                                requiresOpenOrdersSync = true;
+                            }
+                            bot.manager._orphanFillsCreditedAt = Date.now();
+                            continue;
+                        }
+                        const roleStr = fillOp.is_maker !== false ? 'maker' : 'taker';
+                        bot.manager.logger.log(`Processing ${roleStr} fill for order ${fillOp.order_id}`, 'debug');
+                        const fillKey = buildFillKey(fill);
+                        if (!fillKey) {
+                            bot.manager.logger.log(`[FILL] Missing history id for order ${fillOp.order_id} block ${fill.block_num}; deferring to open-orders sync`, 'warn');
+                            requiresOpenOrdersSync = true;
+                            continue;
+                        }
+                        if (!bot._isNewFillKey(fillKey, processedFillKeys, '[FILL]', fillOp.order_id)) {
+                            continue;
+                        }
+                        validFills.push(fill);
+                        const paysAmount = fillOp.pays ? fillOp.pays.amount : '?';
+                        const receivesAmount = fillOp.receives ? fillOp.receives.amount : '?';
+                        bot._log(`\n===== FILL DETECTED =====`);
+                        bot._log(`Order ID: ${fillOp.order_id}`);
+                        bot._log(`Pays: ${paysAmount}, Receives: ${receivesAmount}`);
+                        bot._log(`Block: ${fill.block_num} (History ID: ${fill.id || 'N/A'})`);
+                        bot._log(`=========================\n`);
+                    }
+                }
+                const cleanupTimestamp = Date.now();
+                let cleanedCount = 0;
+                for (const [key, timestamp] of bot._recentlyQueuedFills) {
+                    if (cleanupTimestamp - timestamp > bot._fillDedupeWindowMs) {
+                        bot._recentlyQueuedFills.delete(key);
+                        cleanedCount++;
+                    }
+                }
+                if (cleanedCount > 0) {
+                    bot.manager.logger.log(`Cleaned ${cleanedCount} old queued fill records. Remaining: ${bot._recentlyQueuedFills.size}`, 'debug');
+                }
+                if (validFills.length === 0 && !requiresOpenOrdersSync)
+                    continue;
+                let allFilledOrders = [];
+                let ordersNeedingCorrection = [];
+                const fillMode = chainOrders.getFillProcessingMode();
+                const processValidFills = async (fillsToSync) => {
+                    let resolvedOrders = [];
+                    if (fillMode === 'history') {
+                        bot.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (history mode)`, 'info');
+                        if (fillsToSync.length >= 2) {
+                            const batchResult = await bot.manager.syncFromFillHistoryBatch(fillsToSync, {
+                                persistenceMode: processed_fill_store_1.PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
+                            });
+                            for (const fill of fillsToSync) {
+                                const fillKey = buildFillKey({
+                                    orderId: fill?.op?.[1]?.order_id,
+                                    blockNum: fill?.block_num,
+                                    historyId: fill?.id
+                                });
+                                if (fillKey)
+                                    pendingFillKeysForCurrentCycle.add(fillKey);
+                            }
+                            if (batchResult.filledOrders)
+                                resolvedOrders.push(...batchResult.filledOrders);
+                            if (batchResult.requiresOpenOrdersSync)
+                                requiresOpenOrdersSync = true;
+                        }
+                        else {
+                            for (const fill of fillsToSync) {
+                                const resultHistory = await bot.manager.syncFromFillHistory(fill, {
+                                    persistenceMode: processed_fill_store_1.PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
+                                });
+                                const fillKey = buildFillKey({
+                                    orderId: fill?.op?.[1]?.order_id,
+                                    blockNum: fill?.block_num,
+                                    historyId: fill?.id
+                                });
+                                if (fillKey)
+                                    pendingFillKeysForCurrentCycle.add(fillKey);
+                                if (resultHistory.filledOrders)
+                                    resolvedOrders.push(...resultHistory.filledOrders);
+                                if (resultHistory.requiresOpenOrdersSync)
+                                    requiresOpenOrdersSync = true;
+                            }
+                        }
+                    }
+                    if (fillMode !== 'history' || requiresOpenOrdersSync) {
+                        if (fillMode === 'history' && requiresOpenOrdersSync) {
+                            bot.manager.logger.log('Falling back to open-orders sync for fill(s) missing replay-safe history identifiers', 'warn');
+                        }
+                        bot.manager.logger.log(`Syncing ${fillsToSync.length} fill(s) (open orders mode)`, 'info');
+                        const chainOpenOrders = await chainOrders.readOpenOrders(bot.account);
+                        const resultOpenOrders = await bot.manager.syncFromOpenOrders(chainOpenOrders);
+                        if (resultOpenOrders.filledOrders)
+                            resolvedOrders.push(...resultOpenOrders.filledOrders);
+                        if (resultOpenOrders.ordersNeedingCorrection)
+                            ordersNeedingCorrection.push(...resultOpenOrders.ordersNeedingCorrection);
+                    }
+                    return resolvedOrders;
+                };
+                bot.manager.pauseFundRecalc();
+                try {
+                    const fillsByBlock = new Map();
+                    const fillsWithoutBlock = [];
+                    for (const fill of validFills) {
+                        if (fill.block_num != null) {
+                            const list = fillsByBlock.get(fill.block_num);
+                            if (list)
+                                list.push(fill);
+                            else
+                                fillsByBlock.set(fill.block_num, [fill]);
+                        }
+                        else {
+                            fillsWithoutBlock.push(fill);
+                        }
+                    }
+                    const sortedBlocks = [...fillsByBlock.keys()].sort((a, b) => a - b);
+                    const accumulatedOrders = [];
+                    let anyRequiresSync = false;
+                    const initialRequiresSync = requiresOpenOrdersSync;
+                    for (const blockNum of sortedBlocks) {
+                        requiresOpenOrdersSync = false;
+                        bot.manager.logger.log(`[FILL-BLOCK] Processing ${fillsByBlock.get(blockNum).length} fill(s) from block ${blockNum}`, 'debug');
+                        const blockResult = await processValidFills(fillsByBlock.get(blockNum));
+                        accumulatedOrders.push(...blockResult);
+                        if (requiresOpenOrdersSync)
+                            anyRequiresSync = true;
+                    }
+                    requiresOpenOrdersSync = anyRequiresSync || initialRequiresSync;
+                    if (fillsWithoutBlock.length > 0) {
+                        bot.manager.logger.log(`[FILL-BLOCK] Processing ${fillsWithoutBlock.length} fill(s) without block info`, 'debug');
+                        const noBlockResult = await processValidFills(fillsWithoutBlock);
+                        accumulatedOrders.push(...noBlockResult);
+                        if (requiresOpenOrdersSync)
+                            anyRequiresSync = true;
+                    }
+                    if (requiresOpenOrdersSync && !anyRequiresSync) {
+                        bot.manager.logger.log('[FILL-BLOCK] Running open-orders sync for fills with missing history identifiers', 'warn');
+                        const fallbackOrders = await processValidFills([]);
+                        accumulatedOrders.push(...fallbackOrders);
+                    }
+                    allFilledOrders = accumulatedOrders;
+                    if (ordersNeedingCorrection.length > 0) {
+                        const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders);
+                        if (correctionResult.failed > 0)
+                            bot.manager.logger.log(`${correctionResult.failed} corrections failed`, 'error');
+                    }
+                }
+                finally {
+                    await bot.manager.resumeFundRecalc();
+                }
+                bot._refreshDynamicWeightDistribution('fill queue');
+                if (allFilledOrders.length > 0) {
+                    const result = await bot._processFillsWithBatching(allFilledOrders, null, 'fill set');
+                    let abortedFillCycle = result.aborted;
+                    if (!abortedFillCycle) {
+                        const batchFillKeys = new Set(allFilledOrders.map((filledOrder) => buildFillKey({
+                            orderId: filledOrder?.orderId,
+                            blockNum: filledOrder?.blockNum,
+                            historyId: filledOrder?.historyId
+                        })).filter(Boolean));
+                        await bot._flushProcessedFillPersistenceForKeys(batchFillKeys, 'fill-batch-committed');
+                    }
+                    else {
+                        bot.manager.logger.log('[FILL-DEDUP] Fill cycle aborted; fill key persistence guarded under abort path.', 'warn');
+                    }
+                    const fullFillCount = allFilledOrders.filter((o) => o && o.isPartial !== true).length;
+                    const hasAnyFills = allFilledOrders.some((o) => o);
+                    const shouldRunPostFillChecks = !abortedFillCycle && fullFillCount > 0;
+                    const shouldRunDustDetection = !abortedFillCycle && hasAnyFills;
+                    if (shouldRunDustDetection) {
+                        const healthResult = await bot.manager.checkGridHealth(bot.updateOrdersOnChainPlan.bind(bot));
+                        const allDust = [
+                            ...(healthResult.buyDustOrders || []),
+                            ...(healthResult.sellDustOrders || []),
+                        ];
+                        if (allDust.length > 0) {
+                            const dustCancelResult = await bot._cancelDustOrders({
+                                buy: healthResult.buyDustOrders,
+                                sell: healthResult.sellDustOrders,
+                            });
+                            if (dustCancelResult?.batchResult?.aborted) {
+                                abortedFillCycle = true;
+                            }
+                        }
+                    }
+                    if (shouldRunPostFillChecks && !abortedFillCycle) {
+                        await bot._runGridMaintenance('post-fill');
+                    }
+                }
+                else if (pendingFillKeysForCurrentCycle.size > 0) {
+                    await bot._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, 'fill-batch-no-rotations');
+                }
+                bot.manager._recentFillKeysSnapshot = bot._getRecentFillKeysSnapshot();
+                await retryPersistenceIfNeeded(bot.manager);
+                bot._fillCleanupCounter += validFills.length;
+                const cleanupThreshold = constants_1.MAINTENANCE.CLEANUP_PROBABILITY > 0 && constants_1.MAINTENANCE.CLEANUP_PROBABILITY < 1
+                    ? Math.floor(1 / constants_1.MAINTENANCE.CLEANUP_PROBABILITY)
+                    : 100;
+                if (bot._fillCleanupCounter >= cleanupThreshold) {
+                    try {
+                        await bot.accountOrders.cleanOldProcessedFills(constants_1.TIMING.FILL_RECORD_RETENTION_MS);
+                        bot._fillCleanupCounter = 0;
+                    }
+                    catch (err) {
+                        bot.manager?.logger?.log(`Warning: Fill cleanup failed (will retry): ${(0, errors_1.getErrorMessage)(err)}`, 'warn');
+                    }
+                }
+                bot._metrics.fillsProcessed += validFills.length;
+                bot._metrics.fillProcessingTimeMs += Date.now() - batchStartTime;
+                if (bot._staleCleanedOrderIds.size > 0) {
+                    const now = Date.now();
+                    let prunedCount = 0;
+                    for (const [orderId, markedAt] of bot._staleCleanedOrderIds) {
+                        if (now - markedAt > bot._staleCleanupRetentionMs) {
+                            bot._staleCleanedOrderIds.delete(orderId);
+                            prunedCount++;
+                        }
+                    }
+                    if (prunedCount > 0) {
+                        bot.manager.logger.log(`[STALE-CLEANUP] Pruned ${prunedCount} expired stale-cleaned order IDs ` +
+                            `(retention=${bot._staleCleanupRetentionMs}ms, remaining=${bot._staleCleanedOrderIds.size})`, 'debug');
+                    }
+                }
+            }
+            bot._markGridActivity('fill processing end');
+            bot._consecutiveConsumeFailures = 0;
+            bot._consumeFailureFirstAt = 0;
+        });
+    }
+    catch (err) {
+        const isCredentialOutage = bot._isCredentialDaemonError(err);
+        if (pendingFillKeysForCurrentCycle.size > 0) {
+            const flushReason = isCredentialOutage
+                ? 'credential-outage-verified-fills'
+                : 'fill-cycle-error-verified-fills';
+            if (isCredentialOutage) {
+                bot._credentialRecoveryNeeded = true;
+                bot._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${(0, errors_1.getErrorMessage)(err)}`);
+            }
+            try {
+                await bot._flushProcessedFillPersistenceForKeys(pendingFillKeysForCurrentCycle, flushReason, { throwOnError: true });
+                const credentialSuffix = isCredentialOutage
+                    ? '; grid persistence is suspended until recovery'
+                    : '';
+                bot.manager?.logger?.log?.(`[FILL-DEDUP] Persisted ${pendingFillKeysForCurrentCycle.size} verified processed-fill write(s) after fill cycle error${credentialSuffix}.`, isCredentialOutage ? 'warn' : 'info');
+            }
+            catch (flushErr) {
+                bot.manager?.logger?.log?.(`[FILL-DEDUP] Failed to persist verified fill keys during fill error handling: ${(0, errors_1.getErrorMessage)(flushErr)}`, 'warn');
+            }
+        }
+        if (isCredentialOutage && pendingFillKeysForCurrentCycle.size === 0) {
+            bot._credentialRecoveryNeeded = true;
+            bot._suspendGridPersistenceForCredentialOutage(`credential outage during fill processing: ${(0, errors_1.getErrorMessage)(err)}`);
+        }
+        bot._log(`Error processing fills: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
+        if (err.stack)
+            bot._log(err.stack, 'error');
+    }
+    if (!bot._shuttingDown && bot._incomingFillQueue.length > 0) {
+        scheduleFillConsumerRestart(bot, chainOrders);
+    }
+}
 //# sourceMappingURL=dexbot_fill_runtime.js.map