dexbot 1.5.3 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +75 -0
- package/README.md +9 -7
- package/analysis/README.md +1 -2
- package/analysis/ama_fitting/fetch_lp_candles.ts +1 -9
- package/analysis/ama_fitting/package.json +1 -1
- package/analysis/analyze_derivatives.ts +1 -1
- package/analysis/bot_usage/discover_bot_accounts.ts +12 -6
- package/analysis/resolve_source.ts +9 -2
- package/analysis/tradingview/README.md +29 -11
- package/analysis/tradingview/analyze_tradingview.ts +92 -3
- package/analysis/tradingview/tradingview_uplot_chart_generator.ts +1120 -182
- package/analysis/trend_detection/package.json +1 -1
- package/claw/docs/DEXBOT2_TUNING_CHEAT_SHEET.md +1 -1
- package/claw/modules/dexbot_profiles.ts +22 -2
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/skills/launcher-ops/references/launcher-workflow.md +1 -1
- package/claw/tests/test_claw_mcp_transport.ts +2 -2
- package/claw/tests/test_dexbot_profiles.ts +65 -0
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +1 -9
- package/dist/analysis/ama_fitting/fetch_lp_candles.js.map +1 -1
- package/dist/analysis/analyze_derivatives.js +1 -1
- package/dist/analysis/bot_usage/discover_bot_accounts.js +11 -6
- package/dist/analysis/bot_usage/discover_bot_accounts.js.map +1 -1
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts +2 -0
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts.map +1 -1
- package/dist/analysis/resolve_source.d.ts +1 -0
- package/dist/analysis/resolve_source.d.ts.map +1 -1
- package/dist/analysis/resolve_source.js +6 -2
- package/dist/analysis/resolve_source.js.map +1 -1
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +5 -0
- package/dist/analysis/tradingview/analyze_tradingview.d.ts.map +1 -1
- package/dist/analysis/tradingview/analyze_tradingview.js +112 -3
- package/dist/analysis/tradingview/analyze_tradingview.js.map +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts +5 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts.map +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +1118 -182
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js.map +1 -1
- package/dist/bot.js +1 -1
- package/dist/bot.js.map +1 -1
- package/dist/dexbot.js +4 -4
- package/dist/dexbot.js.map +1 -1
- package/dist/market_adapter/ama_signal_runner.js +2 -2
- package/dist/market_adapter/candle_utils.d.ts +6 -0
- package/dist/market_adapter/candle_utils.d.ts.map +1 -1
- package/dist/market_adapter/candle_utils.js +6 -0
- package/dist/market_adapter/candle_utils.js.map +1 -1
- package/dist/market_adapter/core/kibana_candles.d.ts +17 -1
- package/dist/market_adapter/core/kibana_candles.d.ts.map +1 -1
- package/dist/market_adapter/core/kibana_candles.js +101 -19
- package/dist/market_adapter/core/kibana_candles.js.map +1 -1
- package/dist/market_adapter/core/kibana_client.d.ts +2 -0
- package/dist/market_adapter/core/kibana_client.d.ts.map +1 -1
- package/dist/market_adapter/core/kibana_client.js +31 -4
- package/dist/market_adapter/core/kibana_client.js.map +1 -1
- package/dist/market_adapter/core/market_adapter_service.d.ts.map +1 -1
- package/dist/market_adapter/core/market_adapter_service.js +0 -6
- package/dist/market_adapter/core/market_adapter_service.js.map +1 -1
- package/dist/market_adapter/inputs/fetch_book_data.d.ts +14 -0
- package/dist/market_adapter/inputs/fetch_book_data.d.ts.map +1 -0
- package/dist/market_adapter/inputs/fetch_book_data.js +138 -0
- package/dist/market_adapter/inputs/fetch_book_data.js.map +1 -0
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +1 -1
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +16 -2
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts.map +1 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js +109 -281
- package/dist/market_adapter/inputs/fetch_lp_data.js.map +1 -1
- package/dist/market_adapter/inputs/kibana_feed_source.d.ts +87 -0
- package/dist/market_adapter/inputs/kibana_feed_source.d.ts.map +1 -0
- package/dist/market_adapter/inputs/kibana_feed_source.js +570 -0
- package/dist/market_adapter/inputs/kibana_feed_source.js.map +1 -0
- package/dist/market_adapter/inputs/kibana_source.d.ts.map +1 -1
- package/dist/market_adapter/inputs/kibana_source.js +10 -5
- package/dist/market_adapter/inputs/kibana_source.js.map +1 -1
- package/dist/market_adapter/inputs/window_cache.d.ts +166 -0
- package/dist/market_adapter/inputs/window_cache.d.ts.map +1 -0
- package/dist/market_adapter/inputs/window_cache.js +500 -0
- package/dist/market_adapter/inputs/window_cache.js.map +1 -0
- package/dist/market_adapter/interval_utils.d.ts +8 -1
- package/dist/market_adapter/interval_utils.d.ts.map +1 -1
- package/dist/market_adapter/interval_utils.js +14 -1
- package/dist/market_adapter/interval_utils.js.map +1 -1
- package/dist/modules/account_bots.d.ts +1 -0
- package/dist/modules/account_bots.d.ts.map +1 -1
- package/dist/modules/account_bots.js +21 -5
- package/dist/modules/account_bots.js.map +1 -1
- package/dist/modules/account_orders.d.ts +25 -4
- package/dist/modules/account_orders.d.ts.map +1 -1
- package/dist/modules/account_orders.js +67 -4
- package/dist/modules/account_orders.js.map +1 -1
- package/dist/modules/bitshares-native/chain_client.d.ts +2 -0
- package/dist/modules/bitshares-native/chain_client.d.ts.map +1 -1
- package/dist/modules/bitshares-native/chain_client.js +3 -1
- package/dist/modules/bitshares-native/chain_client.js.map +1 -1
- package/dist/modules/bitshares-native/subscriptions.d.ts.map +1 -1
- package/dist/modules/bitshares-native/subscriptions.js +13 -3
- package/dist/modules/bitshares-native/subscriptions.js.map +1 -1
- package/dist/modules/bitshares-native/transport.d.ts +2 -0
- package/dist/modules/bitshares-native/transport.d.ts.map +1 -1
- package/dist/modules/bitshares-native/transport.js +122 -28
- package/dist/modules/bitshares-native/transport.js.map +1 -1
- package/dist/modules/bitshares_client.d.ts.map +1 -1
- package/dist/modules/bitshares_client.js +59 -8
- package/dist/modules/bitshares_client.js.map +1 -1
- package/dist/modules/bot_settings.d.ts.map +1 -1
- package/dist/modules/bot_settings.js +11 -0
- package/dist/modules/bot_settings.js.map +1 -1
- package/dist/modules/constants.d.ts +18 -0
- package/dist/modules/constants.d.ts.map +1 -1
- package/dist/modules/constants.js +90 -4
- package/dist/modules/constants.js.map +1 -1
- package/dist/modules/dexbot_class.d.ts +11 -0
- package/dist/modules/dexbot_class.d.ts.map +1 -1
- package/dist/modules/dexbot_class.js +92 -4
- package/dist/modules/dexbot_class.js.map +1 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +26 -1
- package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_cow_runtime.js +157 -28
- package/dist/modules/dexbot_cow_runtime.js.map +1 -1
- package/dist/modules/dexbot_fill_runtime.d.ts +66 -1
- package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_fill_runtime.js +295 -10
- package/dist/modules/dexbot_fill_runtime.js.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts +102 -12
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +644 -48
- package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
- package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_startup_runtime.js +13 -0
- package/dist/modules/dexbot_startup_runtime.js.map +1 -1
- package/dist/modules/dexbot_state_recovery.d.ts +15 -1
- package/dist/modules/dexbot_state_recovery.d.ts.map +1 -1
- package/dist/modules/dexbot_state_recovery.js +52 -8
- package/dist/modules/dexbot_state_recovery.js.map +1 -1
- package/dist/modules/fund_registry.d.ts +1 -1
- package/dist/modules/fund_registry.js +1 -1
- package/dist/modules/market_adapter_whitelist.js +1 -1
- package/dist/modules/market_adapter_whitelist.js.map +1 -1
- package/dist/modules/node_connect_policy.d.ts +52 -0
- package/dist/modules/node_connect_policy.d.ts.map +1 -0
- package/dist/modules/node_connect_policy.js +35 -0
- package/dist/modules/node_connect_policy.js.map +1 -0
- package/dist/modules/node_manager.d.ts +32 -0
- package/dist/modules/node_manager.d.ts.map +1 -1
- package/dist/modules/node_manager.js +76 -9
- package/dist/modules/node_manager.js.map +1 -1
- package/dist/modules/order/accounting.d.ts +30 -0
- package/dist/modules/order/accounting.d.ts.map +1 -1
- package/dist/modules/order/accounting.js +176 -3
- package/dist/modules/order/accounting.js.map +1 -1
- package/dist/modules/order/export.d.ts.map +1 -1
- package/dist/modules/order/export.js +1 -0
- package/dist/modules/order/export.js.map +1 -1
- package/dist/modules/order/grid.d.ts.map +1 -1
- package/dist/modules/order/grid.js +189 -25
- package/dist/modules/order/grid.js.map +1 -1
- package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile.js +5 -3
- package/dist/modules/order/grid_reconcile.js.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.js +216 -9
- package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
- package/dist/modules/order/logger.js +1 -1
- package/dist/modules/order/logger.js.map +1 -1
- package/dist/modules/order/manager.d.ts +77 -2
- package/dist/modules/order/manager.d.ts.map +1 -1
- package/dist/modules/order/manager.js +283 -14
- package/dist/modules/order/manager.js.map +1 -1
- package/dist/modules/order/strategy.d.ts +3 -0
- package/dist/modules/order/strategy.d.ts.map +1 -1
- package/dist/modules/order/strategy.js +79 -7
- package/dist/modules/order/strategy.js.map +1 -1
- package/dist/modules/order/sync_engine.d.ts.map +1 -1
- package/dist/modules/order/sync_engine.js +16 -1
- package/dist/modules/order/sync_engine.js.map +1 -1
- package/dist/modules/order/utils/math.d.ts +16 -2
- package/dist/modules/order/utils/math.d.ts.map +1 -1
- package/dist/modules/order/utils/math.js +30 -3
- package/dist/modules/order/utils/math.js.map +1 -1
- package/dist/modules/order/utils/order.d.ts +258 -13
- package/dist/modules/order/utils/order.d.ts.map +1 -1
- package/dist/modules/order/utils/order.js +666 -13
- package/dist/modules/order/utils/order.js.map +1 -1
- package/dist/modules/order/utils/system.d.ts +36 -0
- package/dist/modules/order/utils/system.d.ts.map +1 -1
- package/dist/modules/order/utils/system.js +102 -6
- package/dist/modules/order/utils/system.js.map +1 -1
- package/dist/modules/order/utils/validate.d.ts.map +1 -1
- package/dist/modules/order/utils/validate.js +6 -2
- package/dist/modules/order/utils/validate.js.map +1 -1
- package/dist/modules/paths.d.ts +2 -0
- package/dist/modules/paths.d.ts.map +1 -1
- package/dist/modules/paths.js +1 -0
- package/dist/modules/paths.js.map +1 -1
- package/dist/modules/runtime_settings.d.ts +13 -0
- package/dist/modules/runtime_settings.d.ts.map +1 -1
- package/dist/modules/runtime_settings.js +20 -0
- package/dist/modules/runtime_settings.js.map +1 -1
- package/dist/modules/utils/errors.d.ts +16 -0
- package/dist/modules/utils/errors.d.ts.map +1 -1
- package/dist/modules/utils/errors.js +31 -0
- package/dist/modules/utils/errors.js.map +1 -1
- package/dist/pm2.js +7 -7
- package/dist/scripts/analyze-credit.d.ts.map +1 -1
- package/dist/scripts/analyze-credit.js +13 -3
- package/dist/scripts/analyze-credit.js.map +1 -1
- package/dist/scripts/analyze-orders.js +2 -2
- package/dist/scripts/analyze-orders.js.map +1 -1
- package/dist/scripts/diagnose-kibana-candles.d.ts.map +1 -1
- package/dist/scripts/diagnose-kibana-candles.js +0 -1
- package/dist/scripts/diagnose-kibana-candles.js.map +1 -1
- package/dist/scripts/generate_market_adapter_whitelist.d.ts.map +1 -1
- package/dist/scripts/generate_market_adapter_whitelist.js +3 -2
- package/dist/scripts/generate_market_adapter_whitelist.js.map +1 -1
- package/dist/scripts/test-credit-renewal.js +1 -1
- package/dist/scripts/test-credit-renewal.js.map +1 -1
- package/dist/scripts/tv.d.ts +24 -2
- package/dist/scripts/tv.d.ts.map +1 -1
- package/dist/scripts/tv.js +190 -65
- package/dist/scripts/tv.js.map +1 -1
- package/docs/BITSHARES_ONBOARDING.md +19 -4
- package/docs/COW_INVARIANTS.md +18 -1
- package/docs/DEXBOT_COMPARISON.md +3 -3
- package/docs/EVOLUTION.md +9 -7
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +21 -9
- package/docs/GRID_RECALCULATION.md +13 -13
- package/docs/GRID_RECONCILE.md +12 -2
- package/docs/MPA_CREDIT_USAGE.md +2 -2
- package/docs/README.md +5 -6
- package/docs/architecture.md +0 -58
- package/docs/developer_guide.md +2 -1
- package/docs/docker.md +1 -1
- package/market_adapter/README.md +17 -10
- package/modules/README.md +1 -1
- package/package.json +1 -1
- package/scripts/README.md +9 -7
- package/tests/README.md +2 -2
- package/analysis/results/ama_sweep_results_lp_pool_133_1h.json +0 -2455
- package/analysis/results/bot_fitting_results_lp_pool_133_1h.json +0 -218
- package/analysis/tradingview/h-bts_tradingview.html +0 -1570
- package/analysis/tradingview/t-bts_tradingview.html +0 -1570
|
@@ -188,6 +188,46 @@ declare function maxConsecutiveFillConsumerFailures(bot: any): any;
|
|
|
188
188
|
* @returns {number} Delay in milliseconds before the next retry.
|
|
189
189
|
*/
|
|
190
190
|
declare function computeFillConsumerBackoffMs(bot: any, failures: any): number;
|
|
191
|
+
/**
|
|
192
|
+
* Consume the deferred-drain marker at the start of a fill cycle.
|
|
193
|
+
*
|
|
194
|
+
* When fills were parked by the consumer's broadcast/order-pipeline deferral
|
|
195
|
+
* (or by a region-end / pipeline-clear drain), the cycle that finally runs
|
|
196
|
+
* them deals with ledger deltas that interacted with mid-rebalance grid
|
|
197
|
+
* state. Mirror the orphan-fill credit treatment: stamp
|
|
198
|
+
* manager._orphanFillsCreditedAt so the fund-invariant tolerance is widened
|
|
199
|
+
* (×5) for this cycle only — the drain itself must not trip the invariant.
|
|
200
|
+
* Idempotent: returns true only when a marker was actually consumed.
|
|
201
|
+
* @param {any} bot
|
|
202
|
+
* @returns {boolean} true when the drain marker was consumed (tolerance widened)
|
|
203
|
+
*/
|
|
204
|
+
export declare function consumeDeferredDrainMarker(bot: any): boolean;
|
|
205
|
+
/**
|
|
206
|
+
* Decide whether the fill consumer must defer because a broadcast region is
|
|
207
|
+
* active — WITHOUT acquiring _fillProcessingLock first.
|
|
208
|
+
*
|
|
209
|
+
* Background: a fill-driven rebalance waits up to 30s for broadcast idle
|
|
210
|
+
* INSIDE _fillProcessingLock (20s acquisition timeout). While it sleeps,
|
|
211
|
+
* every concurrent consumer queues as a lock waiter and dies at the 20s
|
|
212
|
+
* timeout ("Error processing fills: Lock acquisition timeout"), cascading
|
|
213
|
+
* under backlog. Deferring before the acquire avoids the sleep entirely:
|
|
214
|
+
* the broadcast's finally plus the region-end hook reschedule the consumer,
|
|
215
|
+
* so freshness is preserved without the wait.
|
|
216
|
+
*
|
|
217
|
+
* Bounded: deferral longer than TIMING.FILL_BROADCAST_DEFER_MAX_MS falls
|
|
218
|
+
* through ({ defer: false, stuckFallback: true }) to the legacy path whose
|
|
219
|
+
* in-lock wait still caps at 30s and proceeds — a leaked flag can delay
|
|
220
|
+
* fills, never starve them. The marker resets whenever the consumer
|
|
221
|
+
* proceeds, so only continuous deferral counts toward the bound.
|
|
222
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
223
|
+
* @param {number} [nowMs] - Clock override (tests)
|
|
224
|
+
* @returns {{ defer: boolean; stuckFallback: boolean }} defer=true means the
|
|
225
|
+
* caller must return without acquiring the lock.
|
|
226
|
+
*/
|
|
227
|
+
declare function shouldDeferFillForBroadcast(bot: any, nowMs?: number): {
|
|
228
|
+
defer: boolean;
|
|
229
|
+
stuckFallback: boolean;
|
|
230
|
+
};
|
|
191
231
|
/**
|
|
192
232
|
* Schedule a fill consumer restart with exponential backoff when the
|
|
193
233
|
* failure budget is exhausted, or immediate retry via setImmediate when
|
|
@@ -204,6 +244,31 @@ declare function scheduleFillConsumerRestart(bot: any, chainOrders: any): void;
|
|
|
204
244
|
* @returns {Promise<void>}
|
|
205
245
|
*/
|
|
206
246
|
declare function processFillsWithBootstrapMode(bot: any, chainOrders: any): Promise<void>;
|
|
247
|
+
/**
|
|
248
|
+
* Park fills deferred on a stale accountTotals refresh for a bounded retry.
|
|
249
|
+
*
|
|
250
|
+
* Background: syncFromFillHistory(Batch) returns { deferred: true } when the
|
|
251
|
+
* accountTotals refresh fails — accounting is NOT applied. The old code
|
|
252
|
+
* dropped those fills (already spliced from the queue) while their dedupe
|
|
253
|
+
* keys stayed marked processed: silent fund loss with a "retrying next
|
|
254
|
+
* cycle" comment, but the next cycle only exists with backlog.
|
|
255
|
+
*
|
|
256
|
+
* Parked fills are held OUTSIDE the live queue (so the maintenance idle gate
|
|
257
|
+
* keeps working) and re-queued on a backoff timer (FILL_TOTALS_RETRY_BASE_MS,
|
|
258
|
+
* doubling, capped at FILL_TOTALS_RETRY_MAX_MS). The retry re-runs the same
|
|
259
|
+
* idempotent sync — safe because nothing was applied and the keys were
|
|
260
|
+
* released. Retries never stop (attempt counter resets on the first cycle
|
|
261
|
+
* with no deferrals); the parked array is capped at MAX_INCOMING_FILL_QUEUE
|
|
262
|
+
* with a critical log as a catastrophic backstop.
|
|
263
|
+
* @param {any} bot
|
|
264
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
265
|
+
* @param {any[]} fills - Raw fill events to park
|
|
266
|
+
* @param {Object} [options]
|
|
267
|
+
* @param {number} [options.retryDelayMs] - Override the computed backoff (tests)
|
|
268
|
+
*/
|
|
269
|
+
export declare function parkFillsForTotalsRetry(bot: any, chainOrders: any, fills: any[], options?: {
|
|
270
|
+
retryDelayMs?: number;
|
|
271
|
+
}): void;
|
|
207
272
|
/**
|
|
208
273
|
* Consume queued fills from incomingFillQueue and rebalance.
|
|
209
274
|
* Deduplicates fills against already-processed set (replay-safe), syncs filled
|
|
@@ -214,5 +279,5 @@ declare function processFillsWithBootstrapMode(bot: any, chainOrders: any): Prom
|
|
|
214
279
|
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
215
280
|
*/
|
|
216
281
|
declare function consumeFillQueue(bot: any, chainOrders: any): Promise<void>;
|
|
217
|
-
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, consumeFillQueue, processFillsWithBootstrapMode };
|
|
282
|
+
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, shouldDeferFillForBroadcast, consumeFillQueue, processFillsWithBootstrapMode };
|
|
218
283
|
//# sourceMappingURL=dexbot_fill_runtime.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dexbot_fill_runtime.d.ts","sourceRoot":"","sources":["../../modules/dexbot_fill_runtime.ts"],"names":[],"mappings":"AAAA,qFAAqF;AAkBrF,UAAU,sBAAsB;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;CACvB;
|
|
1
|
+
{"version":3,"file":"dexbot_fill_runtime.d.ts","sourceRoot":"","sources":["../../modules/dexbot_fill_runtime.ts"],"names":[],"mappings":"AAAA,qFAAqF;AAkBrF,UAAU,sBAAsB;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;CACvB;AA8ED;;;;;;;;;;;;;;;;;GAiBG;AACH,iBAAe,sBAAsB,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAiCnJ;AAED,iBAAS,yBAAyB,CAAC,GAAG,EAAE,GAAG,QAW1C;AAsCD;;;;;;GAMG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,GAAE,GAAc,EAAE,OAAO,GAAE,GAAQ,iBAG/F;AAED;;;;;;;GAOG;AACH,iBAAe,oCAAoC,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAE,GAAuB,EAAE,OAAO,GAAE,GAAQ,iBAG9H;AAED;;;;;;GAMG;AACH,iBAAS,0BAA0B,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,iBAsBtD;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAC3E,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,MAA4B,EAC5B,eAAwB,EACxB,gBAAyB,EACzB,WAAqB,EACrB,eAA4D,EAC5D,sBAA8B,EACjC,GAAE;IACC,iBAAiB,CAAC,EAAE,GAAG,CAAC;IACxB,kBAAkB,CAAC,EAAE,GAAG,CAAC;IACzB,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,GAAG,CAAC;IACtB,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAC/B;;;;;;;;;;;;;;;GAoCL;AAED;;;;;;;;;;;;GAYG;AACH,iBAAe,oCAAoC,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAClF,OAAO,EACP,MAA4B,EAC5B,aAAa,EACb,eAA0D,EAC7D,GAAE;IACC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,eAAe,CAAC,EAAE,GAAG,CAAC;CACpB;;;;;;;;;;;;;;;GAQL;AAED;;;;;;;;;;;;GAYG;AACH,iBAAe,mCAAmC,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EACjF,OAAO,EACP,MAA4B,EAC5B,aAAa,EACb,eAA0D,EAC7D,GAAE;IACC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,eAAe,CAAC,EAAE,GAAG,CAAC;CACpB;;;;;;;;;;;;;;;GAUL;AAED;;;;;;GAMG;AACH,iBAAS,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,IACpC,OAAO,GAAG,mBAmB3B;AAED;;;;GAIG;AACH,iBAAS,kCAAkC,CAAC,GAAG,EAAE,GAAG,OAEnD;AAED;;;;;;;;GAQG;AACH,iBAAS,4BAA4B,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,UAM5D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAO5D;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,aAAa,EAAE,OAAO,CAAA;CAAE,CAoBzG;AAED;;;;;;GAMG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,QA4D9D;AAED;;;;;;GAMG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,iBA0GtE;AAwBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,OAAO,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAO,QAyFxH;AAED;;;;;;;;GAQG;AACH,iBAAe,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,iBAiiBzD;AAED,OAAO,EAAE,yBAAyB,EAAE,6BAA6B,EAAE,oCAAoC,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,oCAAoC,EAAE,mCAAmC,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,kCAAkC,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,gBAAgB,EAAE,6BAA6B,EAAE,CAAA"}
|
|
@@ -6,7 +6,7 @@ import { PROCESSED_FILL_PERSISTENCE_MODES } from './order/processed_fill_store.j
|
|
|
6
6
|
import { NATIVE_CLIENT, FILL_PROCESSING, TIMING, MAINTENANCE, ORDER_TYPES } from './constants.js';
|
|
7
7
|
import { getErrorMessage } from './utils/errors.js';
|
|
8
8
|
import { isOrderDoesNotExistError } from './dexbot_maintenance_runtime.js';
|
|
9
|
-
import { slotIndexForPrice, isSlotInRail } from './order/utils/math.js';
|
|
9
|
+
import { slotIndexForPrice, isChainPriceOutOfGrid, isSlotInRail } from './order/utils/math.js';
|
|
10
10
|
import { ORDER_STATES } from './constants.js';
|
|
11
11
|
function buildFillKey(...args) { return require('./order/utils/order').buildFillKey(...args); }
|
|
12
12
|
function correctAllPriceMismatches(...args) { return require('./order/utils/order').correctAllPriceMismatches(...args); }
|
|
@@ -54,6 +54,18 @@ async function isUnknownFillOrderAdoptable(bot, fillOp) {
|
|
|
54
54
|
if (genesis && Array.isArray(genesis.priceLevels) && genesis.priceLevels.length > 0) {
|
|
55
55
|
try {
|
|
56
56
|
const idx = slotIndexForPrice(price, genesis);
|
|
57
|
+
// Out-of-grid hold (mirror of the sync Pass-2 guard): the
|
|
58
|
+
// clamp onto edge slot 0/N-1 is not a real match. An
|
|
59
|
+
// out-of-grid live order is not adoptable here — fall
|
|
60
|
+
// through to the credit path instead of deferring credit
|
|
61
|
+
// to an adoption sync that must not adopt it.
|
|
62
|
+
{
|
|
63
|
+
const precision = parsed.type === ORDER_TYPES.SELL
|
|
64
|
+
? bot.manager?.assets?.assetA?.precision
|
|
65
|
+
: bot.manager?.assets?.assetB?.precision;
|
|
66
|
+
if (isChainPriceOutOfGrid(price, genesis, precision))
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
57
69
|
const slotId = `slot-${idx}`;
|
|
58
70
|
const slot = bot.manager.orders.get(slotId);
|
|
59
71
|
if (!slot)
|
|
@@ -376,6 +388,76 @@ function computeFillConsumerBackoffMs(bot, failures) {
|
|
|
376
388
|
const stepAfterMax = Math.max(0, failures - maxConsecutiveFillConsumerFailures(bot));
|
|
377
389
|
return Math.min(max, initial * Math.pow(2, stepAfterMax));
|
|
378
390
|
}
|
|
391
|
+
/**
|
|
392
|
+
* Consume the deferred-drain marker at the start of a fill cycle.
|
|
393
|
+
*
|
|
394
|
+
* When fills were parked by the consumer's broadcast/order-pipeline deferral
|
|
395
|
+
* (or by a region-end / pipeline-clear drain), the cycle that finally runs
|
|
396
|
+
* them deals with ledger deltas that interacted with mid-rebalance grid
|
|
397
|
+
* state. Mirror the orphan-fill credit treatment: stamp
|
|
398
|
+
* manager._orphanFillsCreditedAt so the fund-invariant tolerance is widened
|
|
399
|
+
* (×5) for this cycle only — the drain itself must not trip the invariant.
|
|
400
|
+
* Idempotent: returns true only when a marker was actually consumed.
|
|
401
|
+
* @param {any} bot
|
|
402
|
+
* @returns {boolean} true when the drain marker was consumed (tolerance widened)
|
|
403
|
+
*/
|
|
404
|
+
export function consumeDeferredDrainMarker(bot) {
|
|
405
|
+
if (bot && bot._deferredFillsPending) {
|
|
406
|
+
bot._deferredFillsPending = false;
|
|
407
|
+
if (bot.manager)
|
|
408
|
+
bot.manager._orphanFillsCreditedAt = Date.now();
|
|
409
|
+
return true;
|
|
410
|
+
}
|
|
411
|
+
return false;
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Decide whether the fill consumer must defer because a broadcast region is
|
|
415
|
+
* active — WITHOUT acquiring _fillProcessingLock first.
|
|
416
|
+
*
|
|
417
|
+
* Background: a fill-driven rebalance waits up to 30s for broadcast idle
|
|
418
|
+
* INSIDE _fillProcessingLock (20s acquisition timeout). While it sleeps,
|
|
419
|
+
* every concurrent consumer queues as a lock waiter and dies at the 20s
|
|
420
|
+
* timeout ("Error processing fills: Lock acquisition timeout"), cascading
|
|
421
|
+
* under backlog. Deferring before the acquire avoids the sleep entirely:
|
|
422
|
+
* the broadcast's finally plus the region-end hook reschedule the consumer,
|
|
423
|
+
* so freshness is preserved without the wait.
|
|
424
|
+
*
|
|
425
|
+
* Bounded: deferral longer than TIMING.FILL_BROADCAST_DEFER_MAX_MS falls
|
|
426
|
+
* through ({ defer: false, stuckFallback: true }) to the legacy path whose
|
|
427
|
+
* in-lock wait still caps at 30s and proceeds — a leaked flag can delay
|
|
428
|
+
* fills, never starve them. The marker resets whenever the consumer
|
|
429
|
+
* proceeds, so only continuous deferral counts toward the bound.
|
|
430
|
+
* @param {import('./dexbot_class.js').DEXBot} bot
|
|
431
|
+
* @param {number} [nowMs] - Clock override (tests)
|
|
432
|
+
* @returns {{ defer: boolean; stuckFallback: boolean }} defer=true means the
|
|
433
|
+
* caller must return without acquiring the lock.
|
|
434
|
+
*/
|
|
435
|
+
function shouldDeferFillForBroadcast(bot, nowMs) {
|
|
436
|
+
const now = Number.isFinite(Number(nowMs)) ? Number(nowMs) : Date.now();
|
|
437
|
+
let active = false;
|
|
438
|
+
try {
|
|
439
|
+
active = bot?.manager?.isBroadcastingActive?.() === true;
|
|
440
|
+
}
|
|
441
|
+
catch {
|
|
442
|
+
active = false;
|
|
443
|
+
}
|
|
444
|
+
if (!active) {
|
|
445
|
+
if (bot?._fillBroadcastDeferSince)
|
|
446
|
+
bot._fillBroadcastDeferSince = 0;
|
|
447
|
+
return { defer: false, stuckFallback: false };
|
|
448
|
+
}
|
|
449
|
+
const deferMaxMs = Number(TIMING?.FILL_BROADCAST_DEFER_MAX_MS) > 0
|
|
450
|
+
? Number(TIMING.FILL_BROADCAST_DEFER_MAX_MS)
|
|
451
|
+
: 60000;
|
|
452
|
+
const since = Number(bot?._fillBroadcastDeferSince) || 0;
|
|
453
|
+
if (!since)
|
|
454
|
+
bot._fillBroadcastDeferSince = now;
|
|
455
|
+
if (since && (now - since) >= deferMaxMs) {
|
|
456
|
+
bot._fillBroadcastDeferSince = 0;
|
|
457
|
+
return { defer: false, stuckFallback: true };
|
|
458
|
+
}
|
|
459
|
+
return { defer: true, stuckFallback: false };
|
|
460
|
+
}
|
|
379
461
|
/**
|
|
380
462
|
* Schedule a fill consumer restart with exponential backoff when the
|
|
381
463
|
* failure budget is exhausted, or immediate retry via setImmediate when
|
|
@@ -536,6 +618,135 @@ async function processFillsWithBootstrapMode(bot, chainOrders) {
|
|
|
536
618
|
bot.manager.logger.log(`[BOOTSTRAP] Fill error: ${getErrorMessage(err)}`, 'error');
|
|
537
619
|
}
|
|
538
620
|
}
|
|
621
|
+
/**
|
|
622
|
+
* Release replay-safety dedupe keys consumed by _isNewFillKey for fills that
|
|
623
|
+
* are NOT being processed after all (stale-totals deferral). Without this, a
|
|
624
|
+
* re-queued fill is skipped as a duplicate within _fillDedupeWindowMs — the
|
|
625
|
+
* deferral would credit nothing AND poison the retry. Mirrors the
|
|
626
|
+
* orphan-adoption release in processSweepOrphanFill.
|
|
627
|
+
* @param {any} bot
|
|
628
|
+
* @param {Set<any>} processedFillKeys - Per-cycle key set to release from
|
|
629
|
+
* @param {any[]} fills - Raw fill events whose keys were consumed
|
|
630
|
+
*/
|
|
631
|
+
function releaseFillDedupeKeys(bot, processedFillKeys, fills) {
|
|
632
|
+
if (!bot || !processedFillKeys || !Array.isArray(fills))
|
|
633
|
+
return;
|
|
634
|
+
for (const fill of fills) {
|
|
635
|
+
try {
|
|
636
|
+
const key = buildFillKey(fill);
|
|
637
|
+
if (!key)
|
|
638
|
+
continue;
|
|
639
|
+
processedFillKeys.delete(key);
|
|
640
|
+
bot._recentlyQueuedFills?.delete?.(key);
|
|
641
|
+
}
|
|
642
|
+
catch { /* best-effort */ }
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* Park fills deferred on a stale accountTotals refresh for a bounded retry.
|
|
647
|
+
*
|
|
648
|
+
* Background: syncFromFillHistory(Batch) returns { deferred: true } when the
|
|
649
|
+
* accountTotals refresh fails — accounting is NOT applied. The old code
|
|
650
|
+
* dropped those fills (already spliced from the queue) while their dedupe
|
|
651
|
+
* keys stayed marked processed: silent fund loss with a "retrying next
|
|
652
|
+
* cycle" comment, but the next cycle only exists with backlog.
|
|
653
|
+
*
|
|
654
|
+
* Parked fills are held OUTSIDE the live queue (so the maintenance idle gate
|
|
655
|
+
* keeps working) and re-queued on a backoff timer (FILL_TOTALS_RETRY_BASE_MS,
|
|
656
|
+
* doubling, capped at FILL_TOTALS_RETRY_MAX_MS). The retry re-runs the same
|
|
657
|
+
* idempotent sync — safe because nothing was applied and the keys were
|
|
658
|
+
* released. Retries never stop (attempt counter resets on the first cycle
|
|
659
|
+
* with no deferrals); the parked array is capped at MAX_INCOMING_FILL_QUEUE
|
|
660
|
+
* with a critical log as a catastrophic backstop.
|
|
661
|
+
* @param {any} bot
|
|
662
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
663
|
+
* @param {any[]} fills - Raw fill events to park
|
|
664
|
+
* @param {Object} [options]
|
|
665
|
+
* @param {number} [options.retryDelayMs] - Override the computed backoff (tests)
|
|
666
|
+
*/
|
|
667
|
+
export function parkFillsForTotalsRetry(bot, chainOrders, fills, options = {}) {
|
|
668
|
+
if (!bot || !Array.isArray(fills) || fills.length === 0)
|
|
669
|
+
return;
|
|
670
|
+
if (!Array.isArray(bot._fillTotalsParkedFills))
|
|
671
|
+
bot._fillTotalsParkedFills = [];
|
|
672
|
+
const parked = bot._fillTotalsParkedFills;
|
|
673
|
+
const maxParked = Number(NATIVE_CLIENT?.SUBSCRIPTIONS?.MAX_INCOMING_FILL_QUEUE) > 0
|
|
674
|
+
? Number(NATIVE_CLIENT.SUBSCRIPTIONS.MAX_INCOMING_FILL_QUEUE)
|
|
675
|
+
: 1000;
|
|
676
|
+
if (parked.length + fills.length > maxParked) {
|
|
677
|
+
// Cap overflow: drop OLDEST first (parked front, then incoming front)
|
|
678
|
+
// so the surviving set reflects the most current chain state. Either
|
|
679
|
+
// direction is fund-loss without the fund-drift backstop; newest-wins
|
|
680
|
+
// is the safer choice because stale fills are the most likely to have
|
|
681
|
+
// been superseded on-chain.
|
|
682
|
+
const overflow = parked.length + fills.length - maxParked;
|
|
683
|
+
const dropped = [];
|
|
684
|
+
let remaining = overflow;
|
|
685
|
+
if (parked.length > 0 && remaining > 0) {
|
|
686
|
+
const take = Math.min(parked.length, remaining);
|
|
687
|
+
dropped.push(...parked.splice(0, take));
|
|
688
|
+
remaining -= take;
|
|
689
|
+
}
|
|
690
|
+
if (remaining > 0) {
|
|
691
|
+
dropped.push(...fills.splice(0, remaining));
|
|
692
|
+
}
|
|
693
|
+
bot.manager?.logger?.log?.(`[FILL] Totals-refresh retry park overflow (cap ${maxParked}): dropping ${dropped.length} oldest fill(s) ` +
|
|
694
|
+
`(${dropped.map((f) => f?.op?.[1]?.order_id ?? '?').join(',')}); fund-drift detection is the backstop`, 'error');
|
|
695
|
+
if (fills.length === 0)
|
|
696
|
+
return;
|
|
697
|
+
}
|
|
698
|
+
parked.push(...fills);
|
|
699
|
+
// A retry is already scheduled — it picks these up; one timer at a time.
|
|
700
|
+
if (bot._fillTotalsRetryTimer)
|
|
701
|
+
return;
|
|
702
|
+
const attempt = Number(bot._fillTotalsRetryAttempt) || 0;
|
|
703
|
+
const baseMs = Number(TIMING?.FILL_TOTALS_RETRY_BASE_MS) > 0
|
|
704
|
+
? Number(TIMING.FILL_TOTALS_RETRY_BASE_MS)
|
|
705
|
+
: 10000;
|
|
706
|
+
const maxMs = Number(TIMING?.FILL_TOTALS_RETRY_MAX_MS) > 0
|
|
707
|
+
? Number(TIMING.FILL_TOTALS_RETRY_MAX_MS)
|
|
708
|
+
: 60000;
|
|
709
|
+
const delayMs = options?.retryDelayMs
|
|
710
|
+
?? Math.min(baseMs * Math.pow(2, Math.min(attempt, 3)), maxMs);
|
|
711
|
+
bot.manager?.logger?.log?.(`[FILL] Parked ${fills.length} fill(s) on totals-refresh failure ` +
|
|
712
|
+
`(parked=${parked.length}, attempt=${attempt + 1}, retry in ${Math.round(delayMs / 1000)}s); accounting not yet applied`, 'warn');
|
|
713
|
+
bot._fillTotalsRetryTimer = setTimeout(() => {
|
|
714
|
+
bot._fillTotalsRetryTimer = null;
|
|
715
|
+
if (bot._shuttingDown) {
|
|
716
|
+
bot.manager?.logger?.log?.(`[FILL] Shutdown with ${bot._fillTotalsParkedFills?.length || 0} totals-parked fill(s) unprocessed; grid persistence snapshot is the backstop`, 'error');
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
// Re-read the attempt counter fresh at fire time (do NOT reuse the
|
|
720
|
+
// `attempt` captured at park time): a clean cycle may have reset
|
|
721
|
+
// _fillTotalsRetryAttempt to 0 while this timer was pending, and
|
|
722
|
+
// writing back the stale captured value would spuriously inflate the
|
|
723
|
+
// backoff chain. Fresh-read self-corrects; the next park recomputes
|
|
724
|
+
// the delay from the current counter.
|
|
725
|
+
const freshAttempt = Number(bot._fillTotalsRetryAttempt) || 0;
|
|
726
|
+
bot._fillTotalsRetryAttempt = freshAttempt + 1;
|
|
727
|
+
const due = Array.isArray(bot._fillTotalsParkedFills)
|
|
728
|
+
? bot._fillTotalsParkedFills.splice(0)
|
|
729
|
+
: [];
|
|
730
|
+
if (due.length === 0)
|
|
731
|
+
return;
|
|
732
|
+
// NOTE: this unshift runs BEFORE _consumeFillQueue acquires
|
|
733
|
+
// _fillProcessingLock. Interleaving with an in-flight cycle is benign
|
|
734
|
+
// by design: the in-flight run already snapshotted its batch, so the
|
|
735
|
+
// re-queued fills are simply picked up one cycle later (worst case),
|
|
736
|
+
// and _deferredFillsPending widens the invariant tolerance for that
|
|
737
|
+
// drain. Do not move the unshift inside the lock without re-checking
|
|
738
|
+
// the single-flight / bootstrap branches in consumeFillQueue.
|
|
739
|
+
bot._incomingFillQueue.unshift(...due);
|
|
740
|
+
bot._deferredFillsPending = true;
|
|
741
|
+
bot.manager?.logger?.log?.(`[FILL] Retrying ${due.length} totals-parked fill(s) (attempt ${freshAttempt + 1})`, 'warn');
|
|
742
|
+
const consume = typeof bot._consumeFillQueue === 'function'
|
|
743
|
+
? () => bot._consumeFillQueue(chainOrders)
|
|
744
|
+
: () => consumeFillQueue(bot, chainOrders);
|
|
745
|
+
consume().catch((err) => {
|
|
746
|
+
bot._warn?.(`Totals-parked fill retry failed: ${getErrorMessage(err)}`);
|
|
747
|
+
});
|
|
748
|
+
}, delayMs);
|
|
749
|
+
}
|
|
539
750
|
/**
|
|
540
751
|
* Consume queued fills from incomingFillQueue and rebalance.
|
|
541
752
|
* Deduplicates fills against already-processed set (replay-safe), syncs filled
|
|
@@ -552,6 +763,13 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
552
763
|
bot._consumeFailureFirstAt = 0;
|
|
553
764
|
}
|
|
554
765
|
};
|
|
766
|
+
// Deferral marks the queue as a "drain residue": the fills that finally
|
|
767
|
+
// run were parked while the order pipeline / broadcast mutated grid
|
|
768
|
+
// state, so their optimistic ledger interacts with mid-rebalance
|
|
769
|
+
// geometry. The flag is consumed at lock acquire to widen the invariant
|
|
770
|
+
// tolerance like an orphan credit (×5) for that drain cycle only — the
|
|
771
|
+
// drain itself must not trip the fund invariant.
|
|
772
|
+
const markDeferredDrain = () => { bot._deferredFillsPending = true; };
|
|
555
773
|
if (bot._incomingFillQueue.length === 0) {
|
|
556
774
|
resetFailureWatchdogIfSet();
|
|
557
775
|
return;
|
|
@@ -563,10 +781,29 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
563
781
|
}
|
|
564
782
|
if (bot._batchInFlight || bot._recoverySyncInFlight) {
|
|
565
783
|
bot.manager?.logger?.log?.(`Fill processing deferred: order pipeline active (${bot._incomingFillQueue.length} queued)`, 'debug');
|
|
784
|
+
markDeferredDrain();
|
|
566
785
|
resetFailureWatchdogIfSet();
|
|
567
786
|
return;
|
|
568
787
|
}
|
|
788
|
+
// Broadcast pre-check (lock-timeout robustness — see
|
|
789
|
+
// shouldDeferFillForBroadcast): never acquire the fill lock just to
|
|
790
|
+
// sleep in the 30s broadcast wait while concurrent consumers pile up as
|
|
791
|
+
// lock waiters and time out. The broadcast's finally + region-end hook
|
|
792
|
+
// reschedule this consumer when the region ends.
|
|
793
|
+
const broadcastDefer = shouldDeferFillForBroadcast(bot);
|
|
794
|
+
if (broadcastDefer.defer) {
|
|
795
|
+
bot.manager?.logger?.log?.(`Fill processing deferred: broadcast active (${bot._incomingFillQueue.length} queued)`, 'debug');
|
|
796
|
+
markDeferredDrain();
|
|
797
|
+
resetFailureWatchdogIfSet();
|
|
798
|
+
return;
|
|
799
|
+
}
|
|
800
|
+
if (broadcastDefer.stuckFallback) {
|
|
801
|
+
bot.manager?.logger?.log?.('Fill processing proceeding despite stuck broadcast flag (deferral bound exceeded; in-lock wait still applies)', 'warn');
|
|
802
|
+
}
|
|
569
803
|
let pendingFillKeysForCurrentCycle = new Set();
|
|
804
|
+
// Set when this run parked totals-deferred fills: the retry-attempt
|
|
805
|
+
// backoff counter only resets on runs with no deferrals.
|
|
806
|
+
let parkedTotalsRetryThisRun = false;
|
|
570
807
|
try {
|
|
571
808
|
if (bot.manager.isBootstrapping()) {
|
|
572
809
|
let bootstrapSkipped = false;
|
|
@@ -582,13 +819,29 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
582
819
|
}
|
|
583
820
|
return;
|
|
584
821
|
}
|
|
585
|
-
|
|
822
|
+
// Single-flight: getQueueLength() counts WAITERS, not the active
|
|
823
|
+
// holder — without isLocked() every concurrent consumer queues behind
|
|
824
|
+
// a long batch (e.g. a broadcast wait) and dies at the 20s
|
|
825
|
+
// acquisition timeout. The holder's while-loop drains late arrivals
|
|
826
|
+
// (splice-all per iteration + non-empty tail reschedule), so a second
|
|
827
|
+
// consumer is purely redundant — return instead of queueing.
|
|
828
|
+
const fillLock = bot.manager._fillProcessingLock;
|
|
829
|
+
const lockBusy = (typeof fillLock?.isLocked === 'function' && fillLock.isLocked())
|
|
830
|
+
|| (typeof fillLock?.getQueueLength === 'function' && fillLock.getQueueLength() > 0);
|
|
831
|
+
if (lockBusy) {
|
|
586
832
|
bot._metrics.lockContentionEvents++;
|
|
587
833
|
resetFailureWatchdogIfSet();
|
|
588
834
|
return;
|
|
589
835
|
}
|
|
590
836
|
await bot.manager._fillProcessingLock.acquire(async () => {
|
|
591
837
|
bot.manager._orphanFillsCreditedAt = null;
|
|
838
|
+
// Deferred-drain residue tolerance: fills that were parked while a
|
|
839
|
+
// broadcast/order pipeline held the queue are now running against
|
|
840
|
+
// post-rebalance grid state. Mirror the orphan-fill treatment —
|
|
841
|
+
// widen the invariant tolerance for this cycle so the drain itself
|
|
842
|
+
// does not fire a fund-invariant violation. Bounded: cleared at the
|
|
843
|
+
// next cycle start or by the next fresh chain fetch.
|
|
844
|
+
consumeDeferredDrainMarker(bot);
|
|
592
845
|
while (bot._incomingFillQueue.length > 0) {
|
|
593
846
|
const batchStartTime = Date.now();
|
|
594
847
|
bot._metrics.maxQueueDepth = Math.max(bot._metrics.maxQueueDepth, bot._incomingFillQueue.length);
|
|
@@ -674,6 +927,10 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
674
927
|
let allFilledOrders = [];
|
|
675
928
|
let ordersNeedingCorrection = [];
|
|
676
929
|
const residualCancels = [];
|
|
930
|
+
// Fills deferred on a stale accountTotals refresh: accounting
|
|
931
|
+
// NOT applied. Collected here, keys released, parked for a
|
|
932
|
+
// bounded retry after the block loop (never dropped).
|
|
933
|
+
const deferredRefills = [];
|
|
677
934
|
const processValidFills = async (fillsToSync) => {
|
|
678
935
|
let resolvedOrders = [];
|
|
679
936
|
{
|
|
@@ -683,10 +940,15 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
683
940
|
persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
|
|
684
941
|
});
|
|
685
942
|
if (batchResult.deferred) {
|
|
686
|
-
// Deferred: stale accountTotals refresh failed
|
|
687
|
-
//
|
|
688
|
-
//
|
|
689
|
-
|
|
943
|
+
// Deferred: stale accountTotals refresh failed —
|
|
944
|
+
// accounting NOT applied. Release the consumed
|
|
945
|
+
// dedupe keys and park for a bounded retry.
|
|
946
|
+
// (The old code returned here: the fills were
|
|
947
|
+
// already spliced from the queue and their keys
|
|
948
|
+
// stayed marked processed — silent fund loss.)
|
|
949
|
+
releaseFillDedupeKeys(bot, processedFillKeys, fillsToSync);
|
|
950
|
+
deferredRefills.push(...fillsToSync);
|
|
951
|
+
bot.manager.logger.log(`[FILL] Deferred ${fillsToSync.length} fill(s) (stale accountTotals, refresh failed); parking for retry.`, 'warn');
|
|
690
952
|
return resolvedOrders;
|
|
691
953
|
}
|
|
692
954
|
for (const fill of fillsToSync) {
|
|
@@ -716,7 +978,11 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
716
978
|
historyId: fill?.id
|
|
717
979
|
});
|
|
718
980
|
if (resultHistory.deferred) {
|
|
719
|
-
|
|
981
|
+
// Same totals-refresh deferral as the batch
|
|
982
|
+
// path: release keys, park for retry.
|
|
983
|
+
releaseFillDedupeKeys(bot, processedFillKeys, [fill]);
|
|
984
|
+
deferredRefills.push(fill);
|
|
985
|
+
bot.manager.logger.log(`[FILL] Deferred fill (stale accountTotals, refresh failed); parking for retry.`, 'warn');
|
|
720
986
|
continue;
|
|
721
987
|
}
|
|
722
988
|
if (fillKey)
|
|
@@ -793,6 +1059,10 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
793
1059
|
accumulatedOrders.push(...fallbackOrders);
|
|
794
1060
|
}
|
|
795
1061
|
allFilledOrders = accumulatedOrders;
|
|
1062
|
+
if (deferredRefills.length > 0) {
|
|
1063
|
+
parkedTotalsRetryThisRun = true;
|
|
1064
|
+
parkFillsForTotalsRetry(bot, chainOrders, deferredRefills);
|
|
1065
|
+
}
|
|
796
1066
|
if (ordersNeedingCorrection.length > 0) {
|
|
797
1067
|
const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders);
|
|
798
1068
|
if (correctionResult.failed > 0)
|
|
@@ -806,6 +1076,18 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
806
1076
|
if (allFilledOrders.length > 0) {
|
|
807
1077
|
const result = await bot._processFillsWithBatching(allFilledOrders, null, 'fill set');
|
|
808
1078
|
let abortedFillCycle = result.aborted;
|
|
1079
|
+
const deferredFillCycle = result?.deferred === true;
|
|
1080
|
+
if (deferredFillCycle) {
|
|
1081
|
+
// Accounting + crawls are applied; only the broadcast was
|
|
1082
|
+
// deferred to avoid sleeping in-lock on the broadcast
|
|
1083
|
+
// flag. Post-fill maintenance would immediately attempt
|
|
1084
|
+
// the same blocked broadcasts, so skip it here:
|
|
1085
|
+
// _processFillsWithBatching already scheduled the
|
|
1086
|
+
// no-fill rebalance that applies the owed boundary
|
|
1087
|
+
// shift once the pipeline is clear (same idempotent
|
|
1088
|
+
// scheduler the region-end hook uses).
|
|
1089
|
+
bot.manager?.logger?.log?.('[FILL-QUEUE] Fill rebalance deferred (broadcast active); post-fill maintenance deferred to region end', 'debug');
|
|
1090
|
+
}
|
|
809
1091
|
if (!abortedFillCycle) {
|
|
810
1092
|
const batchFillKeys = new Set(allFilledOrders.map((filledOrder) => buildFillKey({
|
|
811
1093
|
orderId: filledOrder?.orderId,
|
|
@@ -827,8 +1109,8 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
827
1109
|
}
|
|
828
1110
|
const fullFillCount = allFilledOrders.filter((o) => o && o.isPartial !== true).length;
|
|
829
1111
|
const hasAnyFills = allFilledOrders.some((o) => o);
|
|
830
|
-
const shouldRunPostFillChecks = !abortedFillCycle && fullFillCount > 0;
|
|
831
|
-
const shouldRunDustDetection = !abortedFillCycle && hasAnyFills;
|
|
1112
|
+
const shouldRunPostFillChecks = !abortedFillCycle && !deferredFillCycle && fullFillCount > 0;
|
|
1113
|
+
const shouldRunDustDetection = !abortedFillCycle && !deferredFillCycle && hasAnyFills;
|
|
832
1114
|
if (shouldRunDustDetection) {
|
|
833
1115
|
const healthResult = await bot.manager.checkGridHealth(bot.updateOrdersOnChainPlan.bind(bot));
|
|
834
1116
|
const allDust = [
|
|
@@ -896,6 +1178,9 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
896
1178
|
}
|
|
897
1179
|
}
|
|
898
1180
|
bot._markGridActivity('fill processing end');
|
|
1181
|
+
// A run with no totals-deferrals breaks the retry backoff chain.
|
|
1182
|
+
if (!parkedTotalsRetryThisRun)
|
|
1183
|
+
bot._fillTotalsRetryAttempt = 0;
|
|
899
1184
|
bot._consecutiveConsumeFailures = 0;
|
|
900
1185
|
bot._consumeFailureFirstAt = 0;
|
|
901
1186
|
});
|
|
@@ -933,5 +1218,5 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
933
1218
|
scheduleFillConsumerRestart(bot, chainOrders);
|
|
934
1219
|
}
|
|
935
1220
|
}
|
|
936
|
-
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, consumeFillQueue, processFillsWithBootstrapMode };
|
|
1221
|
+
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, shouldDeferFillForBroadcast, consumeFillQueue, processFillsWithBootstrapMode };
|
|
937
1222
|
//# sourceMappingURL=dexbot_fill_runtime.js.map
|