dexbot 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +5 -4
- package/analysis/ama_fitting/package.json +1 -1
- package/analysis/tradingview/tradingview_uplot_chart_generator.ts +19 -5
- package/analysis/trend_detection/package.json +1 -1
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/tests/test_claw_mcp_transport.ts +2 -2
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts.map +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +19 -5
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js.map +1 -1
- package/dist/dexbot.js +1 -1
- package/dist/dexbot.js.map +1 -1
- package/dist/market_adapter/inputs/fetch_book_data.js +2 -2
- package/dist/market_adapter/inputs/fetch_book_data.js.map +1 -1
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +22 -3
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts.map +1 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js +15 -15
- package/dist/market_adapter/inputs/fetch_lp_data.js.map +1 -1
- package/dist/market_adapter/inputs/kibana_feed_source.js +2 -2
- package/dist/market_adapter/inputs/kibana_feed_source.js.map +1 -1
- package/dist/market_adapter/inputs/window_cache.d.ts +54 -8
- package/dist/market_adapter/inputs/window_cache.d.ts.map +1 -1
- package/dist/market_adapter/inputs/window_cache.js +328 -87
- package/dist/market_adapter/inputs/window_cache.js.map +1 -1
- package/dist/modules/account_bots.js +2 -2
- package/dist/modules/account_bots.js.map +1 -1
- package/dist/modules/bitshares-native/subscriptions.d.ts +1 -1
- package/dist/modules/bitshares-native/subscriptions.d.ts.map +1 -1
- package/dist/modules/bitshares-native/subscriptions.js +18 -4
- package/dist/modules/bitshares-native/subscriptions.js.map +1 -1
- package/dist/modules/bitshares_client.d.ts.map +1 -1
- package/dist/modules/bitshares_client.js +10 -0
- package/dist/modules/bitshares_client.js.map +1 -1
- package/dist/modules/constants.d.ts +12 -3
- package/dist/modules/constants.d.ts.map +1 -1
- package/dist/modules/constants.js +66 -20
- package/dist/modules/constants.js.map +1 -1
- package/dist/modules/credit_runtime.d.ts.map +1 -1
- package/dist/modules/credit_runtime.js +24 -6
- package/dist/modules/credit_runtime.js.map +1 -1
- package/dist/modules/dexbot_class.d.ts +34 -6
- package/dist/modules/dexbot_class.d.ts.map +1 -1
- package/dist/modules/dexbot_class.js +94 -17
- package/dist/modules/dexbot_class.js.map +1 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +57 -6
- package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_cow_runtime.js +474 -38
- package/dist/modules/dexbot_cow_runtime.js.map +1 -1
- package/dist/modules/dexbot_fill_runtime.d.ts +53 -1
- package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_fill_runtime.js +277 -9
- package/dist/modules/dexbot_fill_runtime.js.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts +69 -0
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +277 -7
- package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
- package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_startup_runtime.js +6 -1
- package/dist/modules/dexbot_startup_runtime.js.map +1 -1
- package/dist/modules/dexbot_state_recovery.d.ts.map +1 -1
- package/dist/modules/dexbot_state_recovery.js +32 -4
- package/dist/modules/dexbot_state_recovery.js.map +1 -1
- package/dist/modules/market_adapter_whitelist.js +1 -1
- package/dist/modules/market_adapter_whitelist.js.map +1 -1
- package/dist/modules/order/grid.d.ts +3 -1
- package/dist/modules/order/grid.d.ts.map +1 -1
- package/dist/modules/order/grid.js +93 -16
- package/dist/modules/order/grid.js.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.js +73 -7
- package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
- package/dist/modules/order/logger.js +1 -1
- package/dist/modules/order/logger.js.map +1 -1
- package/dist/modules/order/manager.d.ts +14 -1
- package/dist/modules/order/manager.d.ts.map +1 -1
- package/dist/modules/order/manager.js +54 -2
- package/dist/modules/order/manager.js.map +1 -1
- package/dist/modules/order/sync_engine.d.ts.map +1 -1
- package/dist/modules/order/sync_engine.js +183 -10
- package/dist/modules/order/sync_engine.js.map +1 -1
- package/dist/modules/order/utils/math.d.ts +65 -15
- package/dist/modules/order/utils/math.d.ts.map +1 -1
- package/dist/modules/order/utils/math.js +93 -22
- package/dist/modules/order/utils/math.js.map +1 -1
- package/dist/modules/order/utils/order.d.ts +51 -1
- package/dist/modules/order/utils/order.d.ts.map +1 -1
- package/dist/modules/order/utils/order.js +122 -2
- package/dist/modules/order/utils/order.js.map +1 -1
- package/dist/modules/utils/errors.d.ts +22 -0
- package/dist/modules/utils/errors.d.ts.map +1 -1
- package/dist/modules/utils/errors.js +30 -0
- package/dist/modules/utils/errors.js.map +1 -1
- package/dist/scripts/generate_market_adapter_whitelist.d.ts.map +1 -1
- package/dist/scripts/generate_market_adapter_whitelist.js +3 -2
- package/dist/scripts/generate_market_adapter_whitelist.js.map +1 -1
- package/docs/BITSHARES_ONBOARDING.md +14 -6
- package/docs/COPY_ON_WRITE_MASTER_PLAN.md +1 -1
- package/docs/DEXBOT_COMPARISON.md +3 -7
- package/docs/EVOLUTION.md +9 -7
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +4 -9
- package/docs/GRID_RECALCULATION.md +8 -7
- package/docs/LIFECYCLE.md +2 -2
- package/docs/README.md +9 -1
- package/docs/architecture.md +7 -7
- package/docs/developer_guide.md +4 -4
- package/market_adapter/README.md +14 -11
- package/package.json +1 -1
- package/scripts/README.md +4 -4
- package/analysis/results/ama_sweep_results_lp_pool_133_1h.json +0 -2455
- package/analysis/results/bot_fitting_results_lp_pool_133_1h.json +0 -218
- package/analysis/tradingview/h-bts_tradingview.html +0 -1570
- package/analysis/tradingview/t-bts_tradingview.html +0 -1570
|
@@ -236,6 +236,33 @@ declare function shouldDeferFillForBroadcast(bot: any, nowMs?: number): {
|
|
|
236
236
|
* @param {Object} chainOrders - Chain orders module
|
|
237
237
|
*/
|
|
238
238
|
declare function scheduleFillConsumerRestart(bot: any, chainOrders: any): void;
|
|
239
|
+
/**
|
|
240
|
+
* Level-triggered retry for fills parked by consumeFillQueue's pre-lock
|
|
241
|
+
* gates (order-pipeline or broadcast deferral).
|
|
242
|
+
*
|
|
243
|
+
* Incident (2026-09-13, H-BTS slot-86): a lone fill deferred by an active
|
|
244
|
+
* broadcast never drained — the defer branches returned without
|
|
245
|
+
* rescheduling, and none of the edge-triggered wake-ups came: the flag was
|
|
246
|
+
* leaked (no region end would ever fire), no second fill arrived to trip
|
|
247
|
+
* the 60s edge-triggered bound, and the non-empty queue held the
|
|
248
|
+
* maintenance idle gate shut (so the maintenance watchdog never ran).
|
|
249
|
+
*
|
|
250
|
+
* This arms a single deduped timer at defer time; on fire it refreshes the
|
|
251
|
+
* stale-broadcast watchdog (outside maintenance) and re-invokes the
|
|
252
|
+
* consumer. While still gated the consumer re-arms, so the loop is
|
|
253
|
+
* self-sustaining until the stuck-fallback bound trips and the queue
|
|
254
|
+
* drains; once drained the timer settles. At most one retry is pending per
|
|
255
|
+
* bot. The timer is unref'd so a parked queue never holds the process open
|
|
256
|
+
* (and fakes that never clear the gate cannot hang the test runner).
|
|
257
|
+
* @param {any} bot
|
|
258
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
259
|
+
* @param {string} reason - Gate label for logging
|
|
260
|
+
* @param {Object} [options]
|
|
261
|
+
* @param {number} [options.retryDelayMs] - Override the derived backoff (tests)
|
|
262
|
+
*/
|
|
263
|
+
declare function scheduleDeferredFillRetry(bot: any, chainOrders: any, reason: string, options?: {
|
|
264
|
+
retryDelayMs?: number;
|
|
265
|
+
}): void;
|
|
239
266
|
/**
|
|
240
267
|
* Process fills during bootstrap phase using the standard fill pipeline.
|
|
241
268
|
* Delegates to the same fill pipeline as the post-reset path.
|
|
@@ -244,6 +271,31 @@ declare function scheduleFillConsumerRestart(bot: any, chainOrders: any): void;
|
|
|
244
271
|
* @returns {Promise<void>}
|
|
245
272
|
*/
|
|
246
273
|
declare function processFillsWithBootstrapMode(bot: any, chainOrders: any): Promise<void>;
|
|
274
|
+
/**
|
|
275
|
+
* Park fills deferred on a stale accountTotals refresh for a bounded retry.
|
|
276
|
+
*
|
|
277
|
+
* Background: syncFromFillHistory(Batch) returns { deferred: true } when the
|
|
278
|
+
* accountTotals refresh fails — accounting is NOT applied. The old code
|
|
279
|
+
* dropped those fills (already spliced from the queue) while their dedupe
|
|
280
|
+
* keys stayed marked processed: silent fund loss with a "retrying next
|
|
281
|
+
* cycle" comment, but the next cycle only exists with backlog.
|
|
282
|
+
*
|
|
283
|
+
* Parked fills are held OUTSIDE the live queue (so the maintenance idle gate
|
|
284
|
+
* keeps working) and re-queued on a backoff timer (FILL_TOTALS_RETRY_BASE_MS,
|
|
285
|
+
* doubling, capped at FILL_TOTALS_RETRY_MAX_MS). The retry re-runs the same
|
|
286
|
+
* idempotent sync — safe because nothing was applied and the keys were
|
|
287
|
+
* released. Retries never stop (attempt counter resets on the first cycle
|
|
288
|
+
* with no deferrals); the parked array is capped at MAX_INCOMING_FILL_QUEUE
|
|
289
|
+
* with a critical log as a catastrophic backstop.
|
|
290
|
+
* @param {any} bot
|
|
291
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
292
|
+
* @param {any[]} fills - Raw fill events to park
|
|
293
|
+
* @param {Object} [options]
|
|
294
|
+
* @param {number} [options.retryDelayMs] - Override the computed backoff (tests)
|
|
295
|
+
*/
|
|
296
|
+
export declare function parkFillsForTotalsRetry(bot: any, chainOrders: any, fills: any[], options?: {
|
|
297
|
+
retryDelayMs?: number;
|
|
298
|
+
}): void;
|
|
247
299
|
/**
|
|
248
300
|
* Consume queued fills from incomingFillQueue and rebalance.
|
|
249
301
|
* Deduplicates fills against already-processed set (replay-safe), syncs filled
|
|
@@ -254,5 +306,5 @@ declare function processFillsWithBootstrapMode(bot: any, chainOrders: any): Prom
|
|
|
254
306
|
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
255
307
|
*/
|
|
256
308
|
declare function consumeFillQueue(bot: any, chainOrders: any): Promise<void>;
|
|
257
|
-
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, shouldDeferFillForBroadcast, consumeFillQueue, processFillsWithBootstrapMode };
|
|
309
|
+
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, scheduleDeferredFillRetry, shouldDeferFillForBroadcast, consumeFillQueue, processFillsWithBootstrapMode };
|
|
258
310
|
//# sourceMappingURL=dexbot_fill_runtime.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dexbot_fill_runtime.d.ts","sourceRoot":"","sources":["../../modules/dexbot_fill_runtime.ts"],"names":[],"mappings":"AAAA,qFAAqF;AAkBrF,UAAU,sBAAsB;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;CACvB;AA8ED;;;;;;;;;;;;;;;;;GAiBG;AACH,iBAAe,sBAAsB,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAiCnJ;AAED,iBAAS,yBAAyB,CAAC,GAAG,EAAE,GAAG,QAW1C;AAsCD;;;;;;GAMG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,GAAE,GAAc,EAAE,OAAO,GAAE,GAAQ,iBAG/F;AAED;;;;;;;GAOG;AACH,iBAAe,oCAAoC,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAE,GAAuB,EAAE,OAAO,GAAE,GAAQ,iBAG9H;AAED;;;;;;GAMG;AACH,iBAAS,0BAA0B,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,iBAsBtD;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAC3E,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,MAA4B,EAC5B,eAAwB,EACxB,gBAAyB,EACzB,WAAqB,EACrB,eAA4D,EAC5D,sBAA8B,EACjC,GAAE;IACC,iBAAiB,CAAC,EAAE,GAAG,CAAC;IACxB,kBAAkB,CAAC,EAAE,GAAG,CAAC;IACzB,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,GAAG,CAAC;IACtB,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAC/B;;;;;;;;;;;;;;;GAoCL;AAED;;;;;;;;;;;;GAYG;AACH,iBAAe,oCAAoC,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAClF,OAAO,EACP,MAA4B,EAC5B,aAAa,EACb,eAA0D,EAC7D,GAAE;IACC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,eAAe,CAAC,EAAE,GAAG,CAAC;CACpB;;;;;;;;;;;;;;;GAQL;AAED;;;;;;;;;;;;GAYG;AACH,iBAAe,mCAAmC,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EACjF,OAAO,EACP,MAA4B,EAC5B,aAAa,EACb,eAA0D,EAC7D,GAAE;IACC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,eAAe,CAAC,EAAE,GAAG,CAAC;CACpB;;;;;;;;;;;;;;;GAUL;AAED;;;;;;GAMG;AACH,iBAAS,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,IACpC,OAAO,GAAG,mBAmB3B;AAED;;;;GAIG;AACH,iBAAS,kCAAkC,CAAC,GAAG,EAAE,GAAG,OAEnD;AAED;;;;;;;;GAQG;AACH,iBAAS,4BAA4B,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,UAM5D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAO5D;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,aAAa,EAAE,OAAO,CAAA;CAAE,CAoBzG;AAED;;;;;;GAMG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,QA4D9D;AAED;;;;;;GAMG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,iBA0GtE;AAED;;;;;;;;GAQG;AACH,iBAAe,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,
|
|
1
|
+
{"version":3,"file":"dexbot_fill_runtime.d.ts","sourceRoot":"","sources":["../../modules/dexbot_fill_runtime.ts"],"names":[],"mappings":"AAAA,qFAAqF;AAkBrF,UAAU,sBAAsB;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;CACvB;AA8ED;;;;;;;;;;;;;;;;;GAiBG;AACH,iBAAe,sBAAsB,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAiCnJ;AAED,iBAAS,yBAAyB,CAAC,GAAG,EAAE,GAAG,QAW1C;AAsCD;;;;;;GAMG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,GAAE,GAAc,EAAE,OAAO,GAAE,GAAQ,iBAG/F;AAED;;;;;;;GAOG;AACH,iBAAe,oCAAoC,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAE,GAAuB,EAAE,OAAO,GAAE,GAAQ,iBAG9H;AAED;;;;;;GAMG;AACH,iBAAS,0BAA0B,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,iBAsBtD;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAC3E,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,MAA4B,EAC5B,eAAwB,EACxB,gBAAyB,EACzB,WAAqB,EACrB,eAA4D,EAC5D,sBAA8B,EACjC,GAAE;IACC,iBAAiB,CAAC,EAAE,GAAG,CAAC;IACxB,kBAAkB,CAAC,EAAE,GAAG,CAAC;IACzB,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,GAAG,CAAC;IACtB,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAC/B;;;;;;;;;;;;;;;GAoCL;AAED;;;;;;;;;;;;GAYG;AACH,iBAAe,oCAAoC,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAClF,OAAO,EACP,MAA4B,EAC5B,aAAa,EACb,eAA0D,EAC7D,GAAE;IACC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,eAAe,CAAC,EAAE,GAAG,CAAC;CACpB;;;;;;;;;;;;;;;GAQL;AAED;;;;;;;;;;;;GAYG;AACH,iBAAe,mCAAmC,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EACjF,OAAO,EACP,MAA4B,EAC5B,aAAa,EACb,eAA0D,EAC7D,GAAE;IACC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,eAAe,CAAC,EAAE,GAAG,CAAC;CACpB;;;;;;;;;;;;;;;GAUL;AAED;;;;;;GAMG;AACH,iBAAS,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,IACpC,OAAO,GAAG,mBAmB3B;AAED;;;;GAIG;AACH,iBAAS,kCAAkC,CAAC,GAAG,EAAE,GAAG,OAEnD;AAED;;;;;;;;GAQG;AACH,iBAAS,4BAA4B,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,UAM5D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAO5D;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,aAAa,EAAE,OAAO,CAAA;CAAE,CAoBzG;AAED;;;;;;GAMG;AACH,iBAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,QA4D9D;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,iBAAS,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAO,QAiDrH;AAED;;;;;;GAMG;AACH,iBAAe,6BAA6B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,iBA0GtE;AAwBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,OAAO,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAO,QA+FxH;AAED;;;;;;;;GAQG;AACH,iBAAe,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,iBA8jBzD;AAED,OAAO,EAAE,yBAAyB,EAAE,6BAA6B,EAAE,oCAAoC,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,oCAAoC,EAAE,mCAAmC,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,kCAAkC,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,gBAAgB,EAAE,6BAA6B,EAAE,CAAA"}
|
|
@@ -4,7 +4,7 @@ const require = createRequire(import.meta.url);
|
|
|
4
4
|
import * as chainOrders from './chain_orders.js';
|
|
5
5
|
import { PROCESSED_FILL_PERSISTENCE_MODES } from './order/processed_fill_store.js';
|
|
6
6
|
import { NATIVE_CLIENT, FILL_PROCESSING, TIMING, MAINTENANCE, ORDER_TYPES } from './constants.js';
|
|
7
|
-
import { getErrorMessage } from './utils/errors.js';
|
|
7
|
+
import { getErrorMessage, resolveSeamMsOrNull } from './utils/errors.js';
|
|
8
8
|
import { isOrderDoesNotExistError } from './dexbot_maintenance_runtime.js';
|
|
9
9
|
import { slotIndexForPrice, isChainPriceOutOfGrid, isSlotInRail } from './order/utils/math.js';
|
|
10
10
|
import { ORDER_STATES } from './constants.js';
|
|
@@ -517,6 +517,85 @@ function scheduleFillConsumerRestart(bot, chainOrders) {
|
|
|
517
517
|
`${remaining} attempts remaining): ${getErrorMessage(err)}`, bot._consecutiveConsumeFailures >= 3 ? 'warn' : 'error');
|
|
518
518
|
}));
|
|
519
519
|
}
|
|
520
|
+
/**
|
|
521
|
+
* Level-triggered retry for fills parked by consumeFillQueue's pre-lock
|
|
522
|
+
* gates (order-pipeline or broadcast deferral).
|
|
523
|
+
*
|
|
524
|
+
* Incident (2026-09-13, H-BTS slot-86): a lone fill deferred by an active
|
|
525
|
+
* broadcast never drained — the defer branches returned without
|
|
526
|
+
* rescheduling, and none of the edge-triggered wake-ups came: the flag was
|
|
527
|
+
* leaked (no region end would ever fire), no second fill arrived to trip
|
|
528
|
+
* the 60s edge-triggered bound, and the non-empty queue held the
|
|
529
|
+
* maintenance idle gate shut (so the maintenance watchdog never ran).
|
|
530
|
+
*
|
|
531
|
+
* This arms a single deduped timer at defer time; on fire it refreshes the
|
|
532
|
+
* stale-broadcast watchdog (outside maintenance) and re-invokes the
|
|
533
|
+
* consumer. While still gated the consumer re-arms, so the loop is
|
|
534
|
+
* self-sustaining until the stuck-fallback bound trips and the queue
|
|
535
|
+
* drains; once drained the timer settles. At most one retry is pending per
|
|
536
|
+
* bot. The timer is unref'd so a parked queue never holds the process open
|
|
537
|
+
* (and fakes that never clear the gate cannot hang the test runner).
|
|
538
|
+
* @param {any} bot
|
|
539
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
540
|
+
* @param {string} reason - Gate label for logging
|
|
541
|
+
* @param {Object} [options]
|
|
542
|
+
* @param {number} [options.retryDelayMs] - Override the derived backoff (tests)
|
|
543
|
+
*/
|
|
544
|
+
function scheduleDeferredFillRetry(bot, chainOrders, reason, options = {}) {
|
|
545
|
+
if (!bot || bot._shuttingDown)
|
|
546
|
+
return;
|
|
547
|
+
if (bot._deferredFillRetryTimer)
|
|
548
|
+
return;
|
|
549
|
+
if (!Array.isArray(bot._incomingFillQueue) || bot._incomingFillQueue.length === 0)
|
|
550
|
+
return;
|
|
551
|
+
const seamRetryMs = resolveSeamMsOrNull(options?.retryDelayMs);
|
|
552
|
+
const retryMs = seamRetryMs ?? (() => {
|
|
553
|
+
const deferMaxMs = Number(TIMING?.FILL_BROADCAST_DEFER_MAX_MS) > 0
|
|
554
|
+
? Number(TIMING.FILL_BROADCAST_DEFER_MAX_MS)
|
|
555
|
+
: 60000;
|
|
556
|
+
return Math.min(5000, Math.max(250, Math.floor(deferMaxMs / 12)));
|
|
557
|
+
})();
|
|
558
|
+
// Expose the resolved delay so tests can assert the seam resolution itself
|
|
559
|
+
// (a `||`-vs-`??` regression here is otherwise invisible: the only
|
|
560
|
+
// observable effect is a slower retry timer, which nothing measures).
|
|
561
|
+
// Only written when resolution is reached — the early guards above
|
|
562
|
+
// (_shuttingDown, timer already armed, empty queue) return before this.
|
|
563
|
+
bot._deferredFillRetryDelayMs = retryMs;
|
|
564
|
+
bot._deferredFillRetryWaits = (Number(bot._deferredFillRetryWaits) || 0) + 1;
|
|
565
|
+
const waits = bot._deferredFillRetryWaits;
|
|
566
|
+
if (waits === 1 || waits % 12 === 0) {
|
|
567
|
+
bot.manager?.logger?.log?.(`[FILL-QUEUE] Fill consumer deferred (${reason}, ${bot._incomingFillQueue.length} queued); ` +
|
|
568
|
+
`retry ${waits} in ${Math.round(retryMs / TIMING.MILLISECONDS_PER_SECOND)}s`, waits === 1 ? 'debug' : 'warn');
|
|
569
|
+
}
|
|
570
|
+
bot._deferredFillRetryTimer = setTimeout(() => {
|
|
571
|
+
bot._deferredFillRetryTimer = null;
|
|
572
|
+
if (bot._shuttingDown)
|
|
573
|
+
return;
|
|
574
|
+
if (!Array.isArray(bot._incomingFillQueue) || bot._incomingFillQueue.length === 0) {
|
|
575
|
+
bot._deferredFillRetryWaits = 0;
|
|
576
|
+
return;
|
|
577
|
+
}
|
|
578
|
+
// Refresh the stale-broadcast watchdog on the retry path, not just
|
|
579
|
+
// in maintenance — a parked queue holds the maintenance idle gate
|
|
580
|
+
// shut, so maintenance can never clear a leaked flag.
|
|
581
|
+
try {
|
|
582
|
+
if (typeof bot.manager?._clearStaleBroadcastFlag === 'function') {
|
|
583
|
+
bot.manager._clearStaleBroadcastFlag();
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
catch { /* watchdog must never break the retry */ }
|
|
587
|
+
const consume = typeof bot._consumeFillQueue === 'function'
|
|
588
|
+
? () => bot._consumeFillQueue(chainOrders)
|
|
589
|
+
: () => consumeFillQueue(bot, chainOrders);
|
|
590
|
+
consume().catch((err) => {
|
|
591
|
+
bot._warn?.(`Deferred fill retry failed: ${getErrorMessage(err)}`);
|
|
592
|
+
});
|
|
593
|
+
}, retryMs);
|
|
594
|
+
try {
|
|
595
|
+
bot._deferredFillRetryTimer?.unref?.();
|
|
596
|
+
}
|
|
597
|
+
catch { /* browser-safe: no unref */ }
|
|
598
|
+
}
|
|
520
599
|
/**
|
|
521
600
|
* Process fills during bootstrap phase using the standard fill pipeline.
|
|
522
601
|
* Delegates to the same fill pipeline as the post-reset path.
|
|
@@ -618,6 +697,141 @@ async function processFillsWithBootstrapMode(bot, chainOrders) {
|
|
|
618
697
|
bot.manager.logger.log(`[BOOTSTRAP] Fill error: ${getErrorMessage(err)}`, 'error');
|
|
619
698
|
}
|
|
620
699
|
}
|
|
700
|
+
/**
|
|
701
|
+
* Release replay-safety dedupe keys consumed by _isNewFillKey for fills that
|
|
702
|
+
* are NOT being processed after all (stale-totals deferral). Without this, a
|
|
703
|
+
* re-queued fill is skipped as a duplicate within _fillDedupeWindowMs — the
|
|
704
|
+
* deferral would credit nothing AND poison the retry. Mirrors the
|
|
705
|
+
* orphan-adoption release in processSweepOrphanFill.
|
|
706
|
+
* @param {any} bot
|
|
707
|
+
* @param {Set<any>} processedFillKeys - Per-cycle key set to release from
|
|
708
|
+
* @param {any[]} fills - Raw fill events whose keys were consumed
|
|
709
|
+
*/
|
|
710
|
+
function releaseFillDedupeKeys(bot, processedFillKeys, fills) {
|
|
711
|
+
if (!bot || !processedFillKeys || !Array.isArray(fills))
|
|
712
|
+
return;
|
|
713
|
+
for (const fill of fills) {
|
|
714
|
+
try {
|
|
715
|
+
const key = buildFillKey(fill);
|
|
716
|
+
if (!key)
|
|
717
|
+
continue;
|
|
718
|
+
processedFillKeys.delete(key);
|
|
719
|
+
bot._recentlyQueuedFills?.delete?.(key);
|
|
720
|
+
}
|
|
721
|
+
catch { /* best-effort */ }
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
/**
|
|
725
|
+
* Park fills deferred on a stale accountTotals refresh for a bounded retry.
|
|
726
|
+
*
|
|
727
|
+
* Background: syncFromFillHistory(Batch) returns { deferred: true } when the
|
|
728
|
+
* accountTotals refresh fails — accounting is NOT applied. The old code
|
|
729
|
+
* dropped those fills (already spliced from the queue) while their dedupe
|
|
730
|
+
* keys stayed marked processed: silent fund loss with a "retrying next
|
|
731
|
+
* cycle" comment, but the next cycle only exists with backlog.
|
|
732
|
+
*
|
|
733
|
+
* Parked fills are held OUTSIDE the live queue (so the maintenance idle gate
|
|
734
|
+
* keeps working) and re-queued on a backoff timer (FILL_TOTALS_RETRY_BASE_MS,
|
|
735
|
+
* doubling, capped at FILL_TOTALS_RETRY_MAX_MS). The retry re-runs the same
|
|
736
|
+
* idempotent sync — safe because nothing was applied and the keys were
|
|
737
|
+
* released. Retries never stop (attempt counter resets on the first cycle
|
|
738
|
+
* with no deferrals); the parked array is capped at MAX_INCOMING_FILL_QUEUE
|
|
739
|
+
* with a critical log as a catastrophic backstop.
|
|
740
|
+
* @param {any} bot
|
|
741
|
+
* @param {Object} chainOrders - Chain orders module for blockchain operations
|
|
742
|
+
* @param {any[]} fills - Raw fill events to park
|
|
743
|
+
* @param {Object} [options]
|
|
744
|
+
* @param {number} [options.retryDelayMs] - Override the computed backoff (tests)
|
|
745
|
+
*/
|
|
746
|
+
export function parkFillsForTotalsRetry(bot, chainOrders, fills, options = {}) {
|
|
747
|
+
if (!bot || !Array.isArray(fills) || fills.length === 0)
|
|
748
|
+
return;
|
|
749
|
+
if (!Array.isArray(bot._fillTotalsParkedFills))
|
|
750
|
+
bot._fillTotalsParkedFills = [];
|
|
751
|
+
const parked = bot._fillTotalsParkedFills;
|
|
752
|
+
const maxParked = Number(NATIVE_CLIENT?.SUBSCRIPTIONS?.MAX_INCOMING_FILL_QUEUE) > 0
|
|
753
|
+
? Number(NATIVE_CLIENT.SUBSCRIPTIONS.MAX_INCOMING_FILL_QUEUE)
|
|
754
|
+
: 1000;
|
|
755
|
+
if (parked.length + fills.length > maxParked) {
|
|
756
|
+
// Cap overflow: drop OLDEST first (parked front, then incoming front)
|
|
757
|
+
// so the surviving set reflects the most current chain state. Either
|
|
758
|
+
// direction is fund-loss without the fund-drift backstop; newest-wins
|
|
759
|
+
// is the safer choice because stale fills are the most likely to have
|
|
760
|
+
// been superseded on-chain.
|
|
761
|
+
const overflow = parked.length + fills.length - maxParked;
|
|
762
|
+
const dropped = [];
|
|
763
|
+
let remaining = overflow;
|
|
764
|
+
if (parked.length > 0 && remaining > 0) {
|
|
765
|
+
const take = Math.min(parked.length, remaining);
|
|
766
|
+
dropped.push(...parked.splice(0, take));
|
|
767
|
+
remaining -= take;
|
|
768
|
+
}
|
|
769
|
+
if (remaining > 0) {
|
|
770
|
+
dropped.push(...fills.splice(0, remaining));
|
|
771
|
+
}
|
|
772
|
+
bot.manager?.logger?.log?.(`[FILL] Totals-refresh retry park overflow (cap ${maxParked}): dropping ${dropped.length} oldest fill(s) ` +
|
|
773
|
+
`(${dropped.map((f) => f?.op?.[1]?.order_id ?? '?').join(',')}); fund-drift detection is the backstop`, 'error');
|
|
774
|
+
if (fills.length === 0)
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
parked.push(...fills);
|
|
778
|
+
// A retry is already scheduled — it picks these up; one timer at a time.
|
|
779
|
+
if (bot._fillTotalsRetryTimer)
|
|
780
|
+
return;
|
|
781
|
+
const attempt = Number(bot._fillTotalsRetryAttempt) || 0;
|
|
782
|
+
const baseMs = Number(TIMING?.FILL_TOTALS_RETRY_BASE_MS) > 0
|
|
783
|
+
? Number(TIMING.FILL_TOTALS_RETRY_BASE_MS)
|
|
784
|
+
: 10000;
|
|
785
|
+
const maxMs = Number(TIMING?.FILL_TOTALS_RETRY_MAX_MS) > 0
|
|
786
|
+
? Number(TIMING.FILL_TOTALS_RETRY_MAX_MS)
|
|
787
|
+
: 60000;
|
|
788
|
+
const delayMs = resolveSeamMsOrNull(options?.retryDelayMs)
|
|
789
|
+
?? Math.min(baseMs * Math.pow(2, Math.min(attempt, 3)), maxMs);
|
|
790
|
+
// Expose the resolved delay so tests can assert the seam resolution itself
|
|
791
|
+
// (a `||`-vs-`??` regression here is otherwise invisible: the only
|
|
792
|
+
// observable effect is a slower retry timer, which nothing measures).
|
|
793
|
+
// Only written when resolution is reached — the early guards above (empty
|
|
794
|
+
// fills, already-armed timer) return before this point.
|
|
795
|
+
bot._fillTotalsRetryDelayMs = delayMs;
|
|
796
|
+
bot.manager?.logger?.log?.(`[FILL] Parked ${fills.length} fill(s) on totals-refresh failure ` +
|
|
797
|
+
`(parked=${parked.length}, attempt=${attempt + 1}, retry in ${Math.round(delayMs / 1000)}s); accounting not yet applied`, 'warn');
|
|
798
|
+
bot._fillTotalsRetryTimer = setTimeout(() => {
|
|
799
|
+
bot._fillTotalsRetryTimer = null;
|
|
800
|
+
if (bot._shuttingDown) {
|
|
801
|
+
bot.manager?.logger?.log?.(`[FILL] Shutdown with ${bot._fillTotalsParkedFills?.length || 0} totals-parked fill(s) unprocessed; grid persistence snapshot is the backstop`, 'error');
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
804
|
+
// Re-read the attempt counter fresh at fire time (do NOT reuse the
|
|
805
|
+
// `attempt` captured at park time): a clean cycle may have reset
|
|
806
|
+
// _fillTotalsRetryAttempt to 0 while this timer was pending, and
|
|
807
|
+
// writing back the stale captured value would spuriously inflate the
|
|
808
|
+
// backoff chain. Fresh-read self-corrects; the next park recomputes
|
|
809
|
+
// the delay from the current counter.
|
|
810
|
+
const freshAttempt = Number(bot._fillTotalsRetryAttempt) || 0;
|
|
811
|
+
bot._fillTotalsRetryAttempt = freshAttempt + 1;
|
|
812
|
+
const due = Array.isArray(bot._fillTotalsParkedFills)
|
|
813
|
+
? bot._fillTotalsParkedFills.splice(0)
|
|
814
|
+
: [];
|
|
815
|
+
if (due.length === 0)
|
|
816
|
+
return;
|
|
817
|
+
// NOTE: this unshift runs BEFORE _consumeFillQueue acquires
|
|
818
|
+
// _fillProcessingLock. Interleaving with an in-flight cycle is benign
|
|
819
|
+
// by design: the in-flight run already snapshotted its batch, so the
|
|
820
|
+
// re-queued fills are simply picked up one cycle later (worst case),
|
|
821
|
+
// and _deferredFillsPending widens the invariant tolerance for that
|
|
822
|
+
// drain. Do not move the unshift inside the lock without re-checking
|
|
823
|
+
// the single-flight / bootstrap branches in consumeFillQueue.
|
|
824
|
+
bot._incomingFillQueue.unshift(...due);
|
|
825
|
+
bot._deferredFillsPending = true;
|
|
826
|
+
bot.manager?.logger?.log?.(`[FILL] Retrying ${due.length} totals-parked fill(s) (attempt ${freshAttempt + 1})`, 'warn');
|
|
827
|
+
const consume = typeof bot._consumeFillQueue === 'function'
|
|
828
|
+
? () => bot._consumeFillQueue(chainOrders)
|
|
829
|
+
: () => consumeFillQueue(bot, chainOrders);
|
|
830
|
+
consume().catch((err) => {
|
|
831
|
+
bot._warn?.(`Totals-parked fill retry failed: ${getErrorMessage(err)}`);
|
|
832
|
+
});
|
|
833
|
+
}, delayMs);
|
|
834
|
+
}
|
|
621
835
|
/**
|
|
622
836
|
* Consume queued fills from incomingFillQueue and rebalance.
|
|
623
837
|
* Deduplicates fills against already-processed set (replay-safe), syncs filled
|
|
@@ -642,6 +856,9 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
642
856
|
// drain itself must not trip the fund invariant.
|
|
643
857
|
const markDeferredDrain = () => { bot._deferredFillsPending = true; };
|
|
644
858
|
if (bot._incomingFillQueue.length === 0) {
|
|
859
|
+
// Settled: retire the deferred-retry counter so the deferral log
|
|
860
|
+
// cadence cannot stay elevated after the queue drained elsewhere.
|
|
861
|
+
bot._deferredFillRetryWaits = 0;
|
|
645
862
|
resetFailureWatchdogIfSet();
|
|
646
863
|
return;
|
|
647
864
|
}
|
|
@@ -650,10 +867,24 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
650
867
|
resetFailureWatchdogIfSet();
|
|
651
868
|
return;
|
|
652
869
|
}
|
|
870
|
+
// Refresh the stale-broadcast watchdog on every consume attempt, not
|
|
871
|
+
// just in maintenance — a leaked flag with a parked queue holds the
|
|
872
|
+
// maintenance idle gate shut, so maintenance would never run to clear
|
|
873
|
+
// it. Guarded: fakes without a manager must still defer cleanly.
|
|
874
|
+
try {
|
|
875
|
+
if (typeof bot.manager?._clearStaleBroadcastFlag === 'function') {
|
|
876
|
+
bot.manager._clearStaleBroadcastFlag();
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
catch { /* watchdog must never break fill consumption */ }
|
|
653
880
|
if (bot._batchInFlight || bot._recoverySyncInFlight) {
|
|
654
881
|
bot.manager?.logger?.log?.(`Fill processing deferred: order pipeline active (${bot._incomingFillQueue.length} queued)`, 'debug');
|
|
655
882
|
markDeferredDrain();
|
|
656
883
|
resetFailureWatchdogIfSet();
|
|
884
|
+
// Level-triggered retry: without this, a lone fill parked here
|
|
885
|
+
// waits for a region end / second fill / maintenance tick that may
|
|
886
|
+
// never come (2026-09-13 H-BTS slot-86).
|
|
887
|
+
scheduleDeferredFillRetry(bot, chainOrders, 'order pipeline active');
|
|
657
888
|
return;
|
|
658
889
|
}
|
|
659
890
|
// Broadcast pre-check (lock-timeout robustness — see
|
|
@@ -666,12 +897,19 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
666
897
|
bot.manager?.logger?.log?.(`Fill processing deferred: broadcast active (${bot._incomingFillQueue.length} queued)`, 'debug');
|
|
667
898
|
markDeferredDrain();
|
|
668
899
|
resetFailureWatchdogIfSet();
|
|
900
|
+
// Level-triggered retry: without this, a lone fill parked here
|
|
901
|
+
// waits for a region end / second fill / maintenance tick that may
|
|
902
|
+
// never come (2026-09-13 H-BTS slot-86).
|
|
903
|
+
scheduleDeferredFillRetry(bot, chainOrders, 'broadcast active');
|
|
669
904
|
return;
|
|
670
905
|
}
|
|
671
906
|
if (broadcastDefer.stuckFallback) {
|
|
672
907
|
bot.manager?.logger?.log?.('Fill processing proceeding despite stuck broadcast flag (deferral bound exceeded; in-lock wait still applies)', 'warn');
|
|
673
908
|
}
|
|
674
909
|
let pendingFillKeysForCurrentCycle = new Set();
|
|
910
|
+
// Set when this run parked totals-deferred fills: the retry-attempt
|
|
911
|
+
// backoff counter only resets on runs with no deferrals.
|
|
912
|
+
let parkedTotalsRetryThisRun = false;
|
|
675
913
|
try {
|
|
676
914
|
if (bot.manager.isBootstrapping()) {
|
|
677
915
|
let bootstrapSkipped = false;
|
|
@@ -795,6 +1033,10 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
795
1033
|
let allFilledOrders = [];
|
|
796
1034
|
let ordersNeedingCorrection = [];
|
|
797
1035
|
const residualCancels = [];
|
|
1036
|
+
// Fills deferred on a stale accountTotals refresh: accounting
|
|
1037
|
+
// NOT applied. Collected here, keys released, parked for a
|
|
1038
|
+
// bounded retry after the block loop (never dropped).
|
|
1039
|
+
const deferredRefills = [];
|
|
798
1040
|
const processValidFills = async (fillsToSync) => {
|
|
799
1041
|
let resolvedOrders = [];
|
|
800
1042
|
{
|
|
@@ -804,10 +1046,15 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
804
1046
|
persistenceMode: PROCESSED_FILL_PERSISTENCE_MODES.BATCHED
|
|
805
1047
|
});
|
|
806
1048
|
if (batchResult.deferred) {
|
|
807
|
-
// Deferred: stale accountTotals refresh failed
|
|
808
|
-
//
|
|
809
|
-
//
|
|
810
|
-
|
|
1049
|
+
// Deferred: stale accountTotals refresh failed —
|
|
1050
|
+
// accounting NOT applied. Release the consumed
|
|
1051
|
+
// dedupe keys and park for a bounded retry.
|
|
1052
|
+
// (The old code returned here: the fills were
|
|
1053
|
+
// already spliced from the queue and their keys
|
|
1054
|
+
// stayed marked processed — silent fund loss.)
|
|
1055
|
+
releaseFillDedupeKeys(bot, processedFillKeys, fillsToSync);
|
|
1056
|
+
deferredRefills.push(...fillsToSync);
|
|
1057
|
+
bot.manager.logger.log(`[FILL] Deferred ${fillsToSync.length} fill(s) (stale accountTotals, refresh failed); parking for retry.`, 'warn');
|
|
811
1058
|
return resolvedOrders;
|
|
812
1059
|
}
|
|
813
1060
|
for (const fill of fillsToSync) {
|
|
@@ -837,7 +1084,11 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
837
1084
|
historyId: fill?.id
|
|
838
1085
|
});
|
|
839
1086
|
if (resultHistory.deferred) {
|
|
840
|
-
|
|
1087
|
+
// Same totals-refresh deferral as the batch
|
|
1088
|
+
// path: release keys, park for retry.
|
|
1089
|
+
releaseFillDedupeKeys(bot, processedFillKeys, [fill]);
|
|
1090
|
+
deferredRefills.push(fill);
|
|
1091
|
+
bot.manager.logger.log(`[FILL] Deferred fill (stale accountTotals, refresh failed); parking for retry.`, 'warn');
|
|
841
1092
|
continue;
|
|
842
1093
|
}
|
|
843
1094
|
if (fillKey)
|
|
@@ -914,6 +1165,10 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
914
1165
|
accumulatedOrders.push(...fallbackOrders);
|
|
915
1166
|
}
|
|
916
1167
|
allFilledOrders = accumulatedOrders;
|
|
1168
|
+
if (deferredRefills.length > 0) {
|
|
1169
|
+
parkedTotalsRetryThisRun = true;
|
|
1170
|
+
parkFillsForTotalsRetry(bot, chainOrders, deferredRefills);
|
|
1171
|
+
}
|
|
917
1172
|
if (ordersNeedingCorrection.length > 0) {
|
|
918
1173
|
const correctionResult = await correctAllPriceMismatches(bot.manager, bot.account, bot.privateKey, chainOrders);
|
|
919
1174
|
if (correctionResult.failed > 0)
|
|
@@ -932,8 +1187,11 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
932
1187
|
// Accounting + crawls are applied; only the broadcast was
|
|
933
1188
|
// deferred to avoid sleeping in-lock on the broadcast
|
|
934
1189
|
// flag. Post-fill maintenance would immediately attempt
|
|
935
|
-
// the same blocked broadcasts, so skip it here
|
|
936
|
-
//
|
|
1190
|
+
// the same blocked broadcasts, so skip it here:
|
|
1191
|
+
// _processFillsWithBatching already scheduled the
|
|
1192
|
+
// no-fill rebalance that applies the owed boundary
|
|
1193
|
+
// shift once the pipeline is clear (same idempotent
|
|
1194
|
+
// scheduler the region-end hook uses).
|
|
937
1195
|
bot.manager?.logger?.log?.('[FILL-QUEUE] Fill rebalance deferred (broadcast active); post-fill maintenance deferred to region end', 'debug');
|
|
938
1196
|
}
|
|
939
1197
|
if (!abortedFillCycle) {
|
|
@@ -1026,6 +1284,9 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
1026
1284
|
}
|
|
1027
1285
|
}
|
|
1028
1286
|
bot._markGridActivity('fill processing end');
|
|
1287
|
+
// A run with no totals-deferrals breaks the retry backoff chain.
|
|
1288
|
+
if (!parkedTotalsRetryThisRun)
|
|
1289
|
+
bot._fillTotalsRetryAttempt = 0;
|
|
1029
1290
|
bot._consecutiveConsumeFailures = 0;
|
|
1030
1291
|
bot._consumeFailureFirstAt = 0;
|
|
1031
1292
|
});
|
|
@@ -1059,9 +1320,16 @@ async function consumeFillQueue(bot, chainOrders) {
|
|
|
1059
1320
|
if (err.stack)
|
|
1060
1321
|
bot._log(err.stack, 'error');
|
|
1061
1322
|
}
|
|
1323
|
+
// Clean settle: a drain that reached the end with an empty queue is the
|
|
1324
|
+
// only place the retry counter is retired — otherwise the elevated count
|
|
1325
|
+
// only matters for log cadence (every 12th defer warns), so reset it here
|
|
1326
|
+
// rather than waiting for a pending retry to fire against an empty queue.
|
|
1327
|
+
if (!bot._incomingFillQueue || bot._incomingFillQueue.length === 0) {
|
|
1328
|
+
bot._deferredFillRetryWaits = 0;
|
|
1329
|
+
}
|
|
1062
1330
|
if (!bot._shuttingDown && bot._incomingFillQueue.length > 0) {
|
|
1063
1331
|
scheduleFillConsumerRestart(bot, chainOrders);
|
|
1064
1332
|
}
|
|
1065
1333
|
}
|
|
1066
|
-
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, shouldDeferFillForBroadcast, consumeFillQueue, processFillsWithBootstrapMode };
|
|
1334
|
+
export { wireProcessedFillTracking, flushProcessedFillPersistence, flushProcessedFillPersistenceForKeys, buildOrphanFillFallbackKey, applyReplaySafeFillAccounting, applyReplaySafeTrackedFillAccounting, applyReplaySafeOrphanFillAccounting, processSweepOrphanFill, createFillCallback, maxConsecutiveFillConsumerFailures, computeFillConsumerBackoffMs, scheduleFillConsumerRestart, scheduleDeferredFillRetry, shouldDeferFillForBroadcast, consumeFillQueue, processFillsWithBootstrapMode };
|
|
1067
1335
|
//# sourceMappingURL=dexbot_fill_runtime.js.map
|