dexbot 1.4.7 → 1.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/credential-daemon.d.ts.map +1 -1
- package/dist/credential-daemon.js +416 -92
- package/dist/credential-daemon.js.map +1 -1
- package/dist/dexbot.d.ts.map +1 -1
- package/dist/dexbot.js +61 -27
- package/dist/dexbot.js.map +1 -1
- package/dist/modules/account_bots.d.ts.map +1 -1
- package/dist/modules/account_bots.js +14 -24
- package/dist/modules/account_bots.js.map +1 -1
- package/dist/modules/bitshares-native/transport.d.ts.map +1 -1
- package/dist/modules/bitshares-native/transport.js +21 -0
- package/dist/modules/bitshares-native/transport.js.map +1 -1
- package/dist/modules/broadcast_failure.d.ts +21 -0
- package/dist/modules/broadcast_failure.d.ts.map +1 -0
- package/dist/modules/broadcast_failure.js +95 -0
- package/dist/modules/broadcast_failure.js.map +1 -0
- package/dist/modules/chain_keys.d.ts +2 -2
- package/dist/modules/chain_keys.d.ts.map +1 -1
- package/dist/modules/chain_keys.js +30 -56
- package/dist/modules/chain_keys.js.map +1 -1
- package/dist/modules/chain_orders.d.ts +79 -1
- package/dist/modules/chain_orders.d.ts.map +1 -1
- package/dist/modules/chain_orders.js +154 -10
- package/dist/modules/chain_orders.js.map +1 -1
- package/dist/modules/constants.d.ts +1 -0
- package/dist/modules/constants.d.ts.map +1 -1
- package/dist/modules/constants.js +21 -7
- package/dist/modules/constants.js.map +1 -1
- package/dist/modules/daemon_node_health.d.ts +33 -0
- package/dist/modules/daemon_node_health.d.ts.map +1 -0
- package/dist/modules/daemon_node_health.js +120 -0
- package/dist/modules/daemon_node_health.js.map +1 -0
- package/dist/modules/dexbot_class.d.ts +12 -3
- package/dist/modules/dexbot_class.d.ts.map +1 -1
- package/dist/modules/dexbot_class.js +19 -5
- package/dist/modules/dexbot_class.js.map +1 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +46 -1
- package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_cow_runtime.js +754 -64
- package/dist/modules/dexbot_cow_runtime.js.map +1 -1
- package/dist/modules/dexbot_credential_client.d.ts +15 -0
- package/dist/modules/dexbot_credential_client.d.ts.map +1 -1
- package/dist/modules/dexbot_credential_client.js +114 -141
- package/dist/modules/dexbot_credential_client.js.map +1 -1
- package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_fill_runtime.js +45 -15
- package/dist/modules/dexbot_fill_runtime.js.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts +5 -1
- package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +237 -84
- package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
- package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -1
- package/dist/modules/dexbot_startup_runtime.js +95 -33
- package/dist/modules/dexbot_startup_runtime.js.map +1 -1
- package/dist/modules/dexbot_state_recovery.d.ts.map +1 -1
- package/dist/modules/dexbot_state_recovery.js +24 -2
- package/dist/modules/dexbot_state_recovery.js.map +1 -1
- package/dist/modules/key_store.d.ts.map +1 -1
- package/dist/modules/key_store.js +5 -4
- package/dist/modules/key_store.js.map +1 -1
- package/dist/modules/launcher/credential_daemon.d.ts.map +1 -1
- package/dist/modules/launcher/credential_daemon.js +15 -3
- package/dist/modules/launcher/credential_daemon.js.map +1 -1
- package/dist/modules/node_failure_ledger.d.ts +39 -0
- package/dist/modules/node_failure_ledger.d.ts.map +1 -0
- package/dist/modules/node_failure_ledger.js +88 -0
- package/dist/modules/node_failure_ledger.js.map +1 -0
- package/dist/modules/node_health_cache.d.ts +2 -1
- package/dist/modules/node_health_cache.d.ts.map +1 -1
- package/dist/modules/node_health_cache.js +25 -0
- package/dist/modules/node_health_cache.js.map +1 -1
- package/dist/modules/node_manager.d.ts +3 -0
- package/dist/modules/node_manager.d.ts.map +1 -1
- package/dist/modules/node_manager.js +41 -10
- package/dist/modules/node_manager.js.map +1 -1
- package/dist/modules/order/accounting.d.ts +1 -1
- package/dist/modules/order/accounting.d.ts.map +1 -1
- package/dist/modules/order/accounting.js +219 -101
- package/dist/modules/order/accounting.js.map +1 -1
- package/dist/modules/order/grid.d.ts.map +1 -1
- package/dist/modules/order/grid.js +66 -14
- package/dist/modules/order/grid.js.map +1 -1
- package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile.js +130 -56
- package/dist/modules/order/grid_reconcile.js.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts +2 -1
- package/dist/modules/order/grid_reconcile_internal.d.ts.map +1 -1
- package/dist/modules/order/grid_reconcile_internal.js +333 -54
- package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
- package/dist/modules/order/manager.d.ts +97 -4
- package/dist/modules/order/manager.d.ts.map +1 -1
- package/dist/modules/order/manager.js +377 -92
- package/dist/modules/order/manager.js.map +1 -1
- package/dist/modules/order/strategy.d.ts.map +1 -1
- package/dist/modules/order/strategy.js +24 -1
- package/dist/modules/order/strategy.js.map +1 -1
- package/dist/modules/order/sync_engine.d.ts +16 -3
- package/dist/modules/order/sync_engine.d.ts.map +1 -1
- package/dist/modules/order/sync_engine.js +329 -247
- package/dist/modules/order/sync_engine.js.map +1 -1
- package/dist/modules/order/utils/math.d.ts +79 -1
- package/dist/modules/order/utils/math.d.ts.map +1 -1
- package/dist/modules/order/utils/math.js +18 -0
- package/dist/modules/order/utils/math.js.map +1 -1
- package/dist/modules/order/utils/order.d.ts +44 -1
- package/dist/modules/order/utils/order.d.ts.map +1 -1
- package/dist/modules/order/utils/order.js +91 -5
- package/dist/modules/order/utils/order.js.map +1 -1
- package/dist/modules/order/utils/system.d.ts +5 -1
- package/dist/modules/order/utils/system.d.ts.map +1 -1
- package/dist/modules/order/utils/system.js +7 -5
- package/dist/modules/order/utils/system.js.map +1 -1
- package/dist/modules/order/utils/withPoolRef.d.ts.map +1 -1
- package/dist/modules/order/utils/withPoolRef.js +39 -4
- package/dist/modules/order/utils/withPoolRef.js.map +1 -1
- package/dist/modules/socket_json_client.d.ts +14 -0
- package/dist/modules/socket_json_client.d.ts.map +1 -0
- package/dist/modules/socket_json_client.js +100 -0
- package/dist/modules/socket_json_client.js.map +1 -0
- package/package.json +2 -1
- package/scripts/clear-all.sh +4 -4
- package/scripts/clear-logs.sh +4 -4
|
@@ -9,16 +9,23 @@
|
|
|
9
9
|
* the same pattern as dexbot_fill_runtime.ts.
|
|
10
10
|
*/
|
|
11
11
|
const chainOrders = require('./chain_orders');
|
|
12
|
+
const { readOpenOrdersWithMetaSafe, readOpenOrdersGuarded } = require('./chain_orders');
|
|
12
13
|
const { BroadcastUncertainError } = require('./dexbot_credential_client');
|
|
13
14
|
const { buildCreateOrderArgs, buildCreateOpFingerprint, extractBatchOperationResults, formatUnmatchedChainOrder, convertToSpreadPlaceholder, buildOutsideInPairGroups, isOrderPlaced, } = require('./order/utils/order');
|
|
14
|
-
const { validateCreateTargetSlots } = require('./order/utils/validate');
|
|
15
|
+
const { validateCreateTargetSlots, evaluateCommit, hasExecutableActions } = require('./order/utils/validate');
|
|
15
16
|
const { validateOrderSize, findPriceCollision } = require('./order/utils/math');
|
|
16
17
|
// Lazy accessor so test mocks on the math module export take effect at call time.
|
|
17
|
-
function
|
|
18
|
+
function getAssetFeesSafe(...args) { return require('./order/utils/math').getAssetFeesSafe(...args); }
|
|
18
19
|
const { COW_ACTIONS, ORDER_STATES, ORDER_TYPES, REBALANCE_STATES, } = require('./constants');
|
|
19
20
|
const Format = require('./order/format');
|
|
20
21
|
const { WorkingGrid } = require('./order/working_grid');
|
|
21
22
|
const { getErrorMessage } = require('./utils/errors');
|
|
23
|
+
// Maximum number of times the pre-broadcast staleness guard may re-plan the
|
|
24
|
+
// batch from a fresh master before proceeding anyway. Bounded so a master
|
|
25
|
+
// grid that keeps mutating (fill bursts, sync loops) can never livelock the
|
|
26
|
+
// pipeline: after one re-plan the batch is shipped regardless, and the
|
|
27
|
+
// commit-time guard + post-refused-commit chain adoption close divergence.
|
|
28
|
+
const STALE_PLAN_REPLAN_LIMIT = 1;
|
|
22
29
|
/**
|
|
23
30
|
* Group orders into outside-in pairs for atomic create execution.
|
|
24
31
|
* @param {Array} orders
|
|
@@ -104,7 +111,19 @@ async function recoverAfterMissingCreateResults(bot, reason = 'missing create op
|
|
|
104
111
|
.filter((order) => order?.reason === 'missing-create-result')
|
|
105
112
|
.map((order) => ({ ...order }))
|
|
106
113
|
: [];
|
|
107
|
-
|
|
114
|
+
// Truncated-read guard: the freshest CREATEs sort last and are exactly
|
|
115
|
+
// the orders a partial get_full_accounts window omits — syncing would
|
|
116
|
+
// virtualize them (phantom cleanup). Defer; blockers stay registered
|
|
117
|
+
// so the COW guard retries the recovery on a clean read.
|
|
118
|
+
const openOrders = await readOpenOrdersGuarded(chainOrders, accountRef, {
|
|
119
|
+
log: (message, level) => bot.manager?.logger?.log?.(message, level),
|
|
120
|
+
label: 'COW',
|
|
121
|
+
detail: `recovery sync after ${reason}`,
|
|
122
|
+
});
|
|
123
|
+
if (openOrders === null) {
|
|
124
|
+
bot.manager?.logger?.log?.(`[COW] Deferring recovery sync after ${reason}: open-order read ambiguous (truncated); blockers retained for retry.`, 'warn');
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
108
127
|
const recoveryResult = await bot.manager.syncFromOpenOrders(openOrders, {
|
|
109
128
|
skipAccounting: false,
|
|
110
129
|
});
|
|
@@ -255,6 +274,69 @@ function clearPendingBroadcasts(pendingBroadcasts) {
|
|
|
255
274
|
pendingBroadcasts.clear();
|
|
256
275
|
}
|
|
257
276
|
}
|
|
277
|
+
/**
|
|
278
|
+
* Drop only the pending-broadcast entries for the given CREATE slots.
|
|
279
|
+
*
|
|
280
|
+
* Used by the re-plan path: the original plan's ops are abandoned with its
|
|
281
|
+
* working grid, so their pending entries must not trip the recursion's own
|
|
282
|
+
* pending-broadcast guard. Entries recorded by an EARLIER unresolved batch
|
|
283
|
+
* (different slots) are KEPT — clearing them here would let the fresh plan
|
|
284
|
+
* re-create slots whose earlier uncertain broadcast may have landed
|
|
285
|
+
* (duplicate orders). The batch-entry guard only fires for batches WITH
|
|
286
|
+
* CREATE actions, so a create-less batch can reach the re-plan path while
|
|
287
|
+
* earlier entries are still live.
|
|
288
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
289
|
+
* @param {Array} actions - The abandoned batch's actions (COW_ACTIONS)
|
|
290
|
+
*/
|
|
291
|
+
function clearPendingBroadcastsForSlots(bot, actions) {
|
|
292
|
+
if (!(bot.manager?._pendingBroadcasts instanceof Map) || !Array.isArray(actions))
|
|
293
|
+
return;
|
|
294
|
+
const slotIds = new Set(actions
|
|
295
|
+
.filter((a) => a?.type === COW_ACTIONS.CREATE)
|
|
296
|
+
.map((a) => a?.id)
|
|
297
|
+
.filter(Boolean));
|
|
298
|
+
if (slotIds.size === 0)
|
|
299
|
+
return;
|
|
300
|
+
for (const [fp, entry] of bot.manager._pendingBroadcasts) {
|
|
301
|
+
if (entry?.slotId && slotIds.has(entry.slotId)) {
|
|
302
|
+
bot.manager._pendingBroadcasts.delete(fp);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Pop a pushed working-grid stack entry exactly once, guarded on the push
|
|
308
|
+
* marker (manager-owned discipline — see OrderManager._pushWorkingGridRef /
|
|
309
|
+
* _popWorkingGridRef). Results that were never pushed (aborted plans,
|
|
310
|
+
* no-trigger processFilledOrders outputs, updateOrdersOnChainPlan cowResults,
|
|
311
|
+
* reconcileGridOrders null results) leave the stack untouched — an unmatched
|
|
312
|
+
* pop could steal a nested grid's entry.
|
|
313
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
314
|
+
* @param {Object} cowResult - Rebalance/COW result carrying _workingGridPushed
|
|
315
|
+
*/
|
|
316
|
+
function popPushedWorkingGrid(bot, cowResult) {
|
|
317
|
+
bot.manager?._popWorkingGridRef?.(cowResult);
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Defer an uncertain-broadcast reconciliation on an ambiguous chain read
|
|
321
|
+
* (empty/truncated/failed). An empty snapshot may be a node lagging behind
|
|
322
|
+
* the just-broadcast transaction and a truncated get_full_accounts window
|
|
323
|
+
* omits the freshest orders (exactly the batch's creates), so absence is
|
|
324
|
+
* never authoritative: the pending-broadcast protection is kept and a
|
|
325
|
+
* structural resync is requested so the next cycle adopts any landed orders.
|
|
326
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
327
|
+
* @param {string} detail - The failure detail (before the common suffix)
|
|
328
|
+
* @param {string} suffix - Parenthetical explanation appended to the message
|
|
329
|
+
* @param {string} resyncReason - Reason string passed to the structural resync
|
|
330
|
+
* @param {Object} [resyncOptions={}] - Extra resync context (batchId, truncated...)
|
|
331
|
+
* @returns {Object} Ambiguous-read reconciliation result
|
|
332
|
+
*/
|
|
333
|
+
async function deferUncertainBroadcastRead(bot, detail, suffix, resyncReason, resyncOptions = {}) {
|
|
334
|
+
bot.manager.logger.log(`[COW][UNCERTAIN] ${detail}; keeping pending-broadcast protection and requesting structural resync ${suffix}`, 'warn');
|
|
335
|
+
if (typeof bot.manager.requestStructuralGridResync === 'function') {
|
|
336
|
+
await bot.manager.requestStructuralGridResync(resyncReason, resyncOptions);
|
|
337
|
+
}
|
|
338
|
+
return { executed: false, hadRotation: false, uncertain: true, ambiguousRead: true };
|
|
339
|
+
}
|
|
258
340
|
/**
|
|
259
341
|
* Build a fingerprint for an on-chain order so it can be matched against
|
|
260
342
|
* the pending-broadcast cache.
|
|
@@ -417,8 +499,8 @@ async function reconcileAfterUncertainBroadcastImpl(bot, err, opContexts, _optio
|
|
|
417
499
|
bot.manager.logger.log(`[COW][UNCERTAIN] batchId=${err?.batchId || 'n/a'} ops=${opContexts.length} ` +
|
|
418
500
|
`creates=${createContextCount} nonCreates=${nonCreateContextCount} ` +
|
|
419
501
|
`staleSinceMs=${err?.timeoutMs || 'n/a'}. Entering reconcile-then-decide.`, 'warn');
|
|
420
|
-
if (!chainOrders?.
|
|
421
|
-
bot.manager.logger.log('[COW][UNCERTAIN]
|
|
502
|
+
if (!chainOrders?.readOpenOrdersWithMeta) {
|
|
503
|
+
bot.manager.logger.log('[COW][UNCERTAIN] readOpenOrdersWithMeta unavailable; falling back to structural resync only.', 'error');
|
|
422
504
|
if (typeof bot.manager.requestStructuralGridResync === 'function') {
|
|
423
505
|
await bot.manager.requestStructuralGridResync('broadcast uncertain — readOpenOrders unavailable', { batchId: err?.batchId || null });
|
|
424
506
|
}
|
|
@@ -428,8 +510,11 @@ async function reconcileAfterUncertainBroadcastImpl(bot, err, opContexts, _optio
|
|
|
428
510
|
// 1. Read the chain
|
|
429
511
|
const accountRef = bot.accountId || bot.account?.id || bot.account;
|
|
430
512
|
let chainSnapshot = [];
|
|
513
|
+
let chainReadTruncated = false;
|
|
431
514
|
try {
|
|
432
|
-
|
|
515
|
+
const chainRead = await chainOrders.readOpenOrdersWithMeta(accountRef);
|
|
516
|
+
chainSnapshot = chainRead.orders;
|
|
517
|
+
chainReadTruncated = chainRead.truncated;
|
|
433
518
|
}
|
|
434
519
|
catch (readErr) {
|
|
435
520
|
bot.manager.logger.log(`[COW][UNCERTAIN] readOpenOrders failed: ${readErr?.message || readErr}. ` +
|
|
@@ -440,6 +525,19 @@ async function reconcileAfterUncertainBroadcastImpl(bot, err, opContexts, _optio
|
|
|
440
525
|
clearPendingBroadcasts(bot.manager?._pendingBroadcasts);
|
|
441
526
|
return { executed: false, hadRotation: false, uncertain: true };
|
|
442
527
|
}
|
|
528
|
+
// 1.5. Empty/truncated-read guard: an empty snapshot is ambiguous — the
|
|
529
|
+
// account is either genuinely empty or the node is lagging behind the
|
|
530
|
+
// just-broadcast transaction. A truncated snapshot (get_full_accounts
|
|
531
|
+
// capped limit_orders; fresh creates sort last in the by_account index
|
|
532
|
+
// and are the first entries omitted) is equally ambiguous: the batch's
|
|
533
|
+
// creates may simply be missing from the returned window. Treating every
|
|
534
|
+
// pending broadcast as discarded would clear the pending-broadcast
|
|
535
|
+
// protection and let the next cycle re-CREATE slots whose orders may
|
|
536
|
+
// actually be on chain (duplicate orders). Keep the protection and let
|
|
537
|
+
// the structural resync adopt any landed orders.
|
|
538
|
+
if (pending.length > 0 && (chainSnapshot.length === 0 || chainReadTruncated)) {
|
|
539
|
+
return await deferUncertainBroadcastRead(bot, `${chainSnapshot.length === 0 ? 'Empty' : 'Truncated'} chain read for ${pending.length} pending broadcast(s)`, '(node may be lagging or the result set capped; no discard decisions made)', 'uncertain broadcast — empty/truncated chain read', { batchId: err?.batchId || null, truncated: chainReadTruncated });
|
|
540
|
+
}
|
|
443
541
|
const adopted = [];
|
|
444
542
|
let discarded = [];
|
|
445
543
|
// 2. For each pending broadcast, look for a chain match.
|
|
@@ -488,7 +586,21 @@ async function reconcileAfterUncertainBroadcastImpl(bot, err, opContexts, _optio
|
|
|
488
586
|
});
|
|
489
587
|
if (createDiscarded.length > 0) {
|
|
490
588
|
try {
|
|
491
|
-
const
|
|
589
|
+
const freshRead = await readOpenOrdersWithMetaSafe(chainOrders, accountRef);
|
|
590
|
+
// An empty/truncated re-read is as ambiguous as the initial
|
|
591
|
+
// read: a truncated get_full_accounts window omits the freshest
|
|
592
|
+
// creates (exactly the discarded ones being re-verified), and an
|
|
593
|
+
// empty snapshot may be a node lagging behind the just-broadcast
|
|
594
|
+
// transaction. Absence in either case is NOT authoritative —
|
|
595
|
+
// discarding here would free the slot + clear the pending
|
|
596
|
+
// protection and let the next cycle re-create (duplicate) an
|
|
597
|
+
// order that actually landed in the TOCTOU window. Keep the
|
|
598
|
+
// pending-broadcast protection and defer to a structural resync.
|
|
599
|
+
if (!freshRead || freshRead.truncated || !Array.isArray(freshRead.orders) || freshRead.orders.length === 0) {
|
|
600
|
+
const ambiguous = !freshRead || !Array.isArray(freshRead.orders) || freshRead.orders.length === 0;
|
|
601
|
+
return await deferUncertainBroadcastRead(bot, `${ambiguous ? 'Empty' : 'Truncated'} re-read for ${createDiscarded.length} discarded CREATE(s)`, '(absence is not authoritative on an ambiguous re-read)', 'uncertain broadcast — ambiguous re-read for discarded creates', { batchId: err?.batchId || null, truncated: !ambiguous });
|
|
602
|
+
}
|
|
603
|
+
const freshChain = freshRead.orders;
|
|
492
604
|
const remainingDiscarded = [];
|
|
493
605
|
for (const entry of discarded) {
|
|
494
606
|
const ctx = opContexts[entry.ctxIndex];
|
|
@@ -513,8 +625,8 @@ async function reconcileAfterUncertainBroadcastImpl(bot, err, opContexts, _optio
|
|
|
513
625
|
}
|
|
514
626
|
discarded = remainingDiscarded;
|
|
515
627
|
}
|
|
516
|
-
catch {
|
|
517
|
-
bot
|
|
628
|
+
catch (reReadErr) {
|
|
629
|
+
return await deferUncertainBroadcastRead(bot, `Fresh chain read for late adoption FAILED (${getErrorMessage(reReadErr)})`, '(absence is not authoritative on a failed re-read)', 'uncertain broadcast — failed re-read for discarded creates', { batchId: err?.batchId || null });
|
|
518
630
|
}
|
|
519
631
|
}
|
|
520
632
|
}
|
|
@@ -528,12 +640,12 @@ async function reconcileAfterUncertainBroadcastImpl(bot, err, opContexts, _optio
|
|
|
528
640
|
const chainOrderId = match.id;
|
|
529
641
|
const expectedType = plannedOpCtx.order?.type || entry.orderType;
|
|
530
642
|
try {
|
|
531
|
-
const btsFeeData =
|
|
643
|
+
const btsFeeData = getAssetFeesSafe('BTS');
|
|
532
644
|
await bot.manager.synchronizeWithChain({
|
|
533
645
|
gridOrderId: plannedOpCtx.order?.id || entry.slotId,
|
|
534
646
|
chainOrderId,
|
|
535
647
|
expectedType,
|
|
536
|
-
fee: btsFeeData
|
|
648
|
+
fee: btsFeeData?.createFee || 0,
|
|
537
649
|
}, 'createOrder');
|
|
538
650
|
}
|
|
539
651
|
catch (syncErr) {
|
|
@@ -558,14 +670,52 @@ async function reconcileAfterUncertainBroadcastImpl(bot, err, opContexts, _optio
|
|
|
558
670
|
if (entry.order?.id && entry.order?.size && entry.order?.type) {
|
|
559
671
|
const slot = bot.manager.orders.get(entry.order.id);
|
|
560
672
|
if (slot) {
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
673
|
+
const plannedType = entry.order.type;
|
|
674
|
+
if (plannedType === ORDER_TYPES.BUY || plannedType === ORDER_TYPES.SELL) {
|
|
675
|
+
// Creation-uncertain state: the broadcast MAY have
|
|
676
|
+
// landed on chain even though no match was found yet.
|
|
677
|
+
// Keep the planned type and size on the slot (VIRTUAL)
|
|
678
|
+
// instead of restoring the SPREAD placeholder, whose
|
|
679
|
+
// size is normalized to 0 by the SPREAD invariant.
|
|
680
|
+
// A possibly-landed order must never be released as a
|
|
681
|
+
// clean hole — that frees the slot for a duplicate
|
|
682
|
+
// CREATE and later orphan adoption double-commits the
|
|
683
|
+
// funds. The next sync's orphan adoption reconciles a
|
|
684
|
+
// landed order into this slot cleanly.
|
|
685
|
+
bot.manager.logger.log(`[COW][UNCERTAIN] Restored creation-uncertain state for slot ${entry.slotId} ` +
|
|
686
|
+
`(type=${plannedType}, size: ${entry.order.size}); next sync adoption will reconcile any landed order`, 'warn');
|
|
687
|
+
const updates = [{
|
|
688
|
+
...slot,
|
|
689
|
+
type: plannedType,
|
|
690
|
+
size: entry.order.size,
|
|
691
|
+
price: entry.order.price,
|
|
692
|
+
state: ORDER_STATES.VIRTUAL,
|
|
693
|
+
// Clear any stale order identity: the broadcast
|
|
694
|
+
// MAY have landed, but the slot must look like a
|
|
695
|
+
// clean adoption target (no orderId/rawOnChain)
|
|
696
|
+
// so the next sync's orphan adoption can reconcile
|
|
697
|
+
// a landed order into it. A retained orderId would
|
|
698
|
+
// make pass-2 adoption skip the slot (it requires
|
|
699
|
+
// !adoptedSlot.orderId), leaving the landed order
|
|
700
|
+
// unmatched and auto-cancelled; a stale rawOnChain
|
|
701
|
+
// would feed a bogus drift signal.
|
|
702
|
+
orderId: null,
|
|
703
|
+
rawOnChain: null,
|
|
704
|
+
}];
|
|
705
|
+
if (typeof bot.manager.applyGridUpdateBatch === 'function') {
|
|
706
|
+
await bot.manager.applyGridUpdateBatch(updates, 'uncertain-broadcast-discard-restore');
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
else {
|
|
710
|
+
bot.manager.logger.log(`[COW][UNCERTAIN] Restored target size for discarded CREATE slot ${entry.slotId} (size: ${entry.order.size})`, 'debug');
|
|
711
|
+
const updates = [{
|
|
712
|
+
...slot,
|
|
713
|
+
size: entry.order.size,
|
|
714
|
+
price: entry.order.price,
|
|
715
|
+
}];
|
|
716
|
+
if (typeof bot.manager.applyGridUpdateBatch === 'function') {
|
|
717
|
+
await bot.manager.applyGridUpdateBatch(updates, 'uncertain-broadcast-discard-restore');
|
|
718
|
+
}
|
|
569
719
|
}
|
|
570
720
|
}
|
|
571
721
|
}
|
|
@@ -718,8 +868,102 @@ function shouldExecuteCreatePairMode(_bot, opContexts) {
|
|
|
718
868
|
}
|
|
719
869
|
return false;
|
|
720
870
|
}
|
|
871
|
+
/**
|
|
872
|
+
* Verify one op context against a fresh chain snapshot for pre-retry
|
|
873
|
+
* re-broadcast safety. Verdicts per kind (see the kind-specific verifiers):
|
|
874
|
+
* - 'absent' → provably never transmitted → retry safe
|
|
875
|
+
* - 'landed' → provably applied on chain → must defer
|
|
876
|
+
* - 'unknown' → chain state unverifiable → must defer
|
|
877
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
878
|
+
* @param {Array} freshChain - Non-empty, non-truncated chain snapshot
|
|
879
|
+
* @param {Object} ctx - Operation context (kind: create/cancel/size-update/rotation)
|
|
880
|
+
* @returns {'absent' | 'landed' | 'unknown'}
|
|
881
|
+
*/
|
|
882
|
+
function verifyOpAgainstChain(bot, freshChain, ctx) {
|
|
883
|
+
if (ctx.kind === 'create')
|
|
884
|
+
return verifyCreateAbsent(bot, freshChain, ctx);
|
|
885
|
+
if (ctx.kind === 'cancel')
|
|
886
|
+
return verifyCancelLanded(freshChain, ctx);
|
|
887
|
+
if (ctx.kind === 'size-update' || ctx.kind === 'rotation')
|
|
888
|
+
return verifyUpdateUnapplied(freshChain, ctx);
|
|
889
|
+
return 'unknown';
|
|
890
|
+
}
|
|
891
|
+
/**
|
|
892
|
+
* CREATE verify: 'absent' only when the batch's creates are found NOWHERE in
|
|
893
|
+
* the snapshot (fingerprint/near-match) — never transmitted → retry safe.
|
|
894
|
+
* Any match means the broadcast landed ('landed'). A create without
|
|
895
|
+
* fingerprint data cannot match anything, so it is treated as absent
|
|
896
|
+
* (original semantics).
|
|
897
|
+
*/
|
|
898
|
+
function verifyCreateAbsent(bot, freshChain, ctx) {
|
|
899
|
+
if (!ctx.finalInts || !ctx.order)
|
|
900
|
+
return 'absent';
|
|
901
|
+
const match = findChainOrderForSlot(bot, freshChain, ctx.order.id, {
|
|
902
|
+
sell: ctx.finalInts.sell,
|
|
903
|
+
receive: ctx.finalInts.receive,
|
|
904
|
+
orderType: ctx.order.type,
|
|
905
|
+
fingerprint: buildCreateOpFingerprint({
|
|
906
|
+
side: ctx.order.type,
|
|
907
|
+
assetA: bot.manager?.assets?.assetA?.id,
|
|
908
|
+
assetB: bot.manager?.assets?.assetB?.id,
|
|
909
|
+
sellInt: ctx.finalInts.sell,
|
|
910
|
+
receiveInt: ctx.finalInts.receive,
|
|
911
|
+
slotId: ctx.order.id
|
|
912
|
+
})
|
|
913
|
+
});
|
|
914
|
+
return match ? 'landed' : 'absent';
|
|
915
|
+
}
|
|
916
|
+
/**
|
|
917
|
+
* CANCEL verify: the order still present → the cancel never landed ('absent',
|
|
918
|
+
* retry safe). Absent from a live snapshot → the cancel landed ('landed').
|
|
919
|
+
* No orderId → unverifiable ('unknown').
|
|
920
|
+
*/
|
|
921
|
+
function verifyCancelLanded(freshChain, ctx) {
|
|
922
|
+
const chainOrderId = ctx.order?.orderId;
|
|
923
|
+
if (!chainOrderId)
|
|
924
|
+
return 'unknown';
|
|
925
|
+
if (!freshChain.some((o) => String(o?.id ?? '') === String(chainOrderId))) {
|
|
926
|
+
return 'landed';
|
|
927
|
+
}
|
|
928
|
+
return 'absent';
|
|
929
|
+
}
|
|
930
|
+
/**
|
|
931
|
+
* UPDATE verify (size-update/rotation): limit_order_update ops are DELTAS, so
|
|
932
|
+
* a landed broadcast double-applies the size change on re-broadcast. Retry
|
|
933
|
+
* ('absent') only when the chain order is provably UNCHANGED from the
|
|
934
|
+
* pre-update cache (the update never applied). Target applied, partially
|
|
935
|
+
* filled after a landed update, or the order missing (filled/cancelled
|
|
936
|
+
* concurrently) → 'unknown' (defer).
|
|
937
|
+
*/
|
|
938
|
+
function verifyUpdateUnapplied(freshChain, ctx) {
|
|
939
|
+
const chainOrderId = ctx.kind === 'size-update'
|
|
940
|
+
? ctx.updateInfo?.partialOrder?.orderId
|
|
941
|
+
: ctx.rotation?.oldOrder?.orderId;
|
|
942
|
+
const cachedRaw = ctx.kind === 'size-update'
|
|
943
|
+
? ctx.updateInfo?.partialOrder?.rawOnChain
|
|
944
|
+
: ctx.rotation?.oldOrder?.rawOnChain;
|
|
945
|
+
if (!chainOrderId)
|
|
946
|
+
return 'unknown';
|
|
947
|
+
const chainOrder = freshChain.find((o) => String(o?.id ?? '') === String(chainOrderId));
|
|
948
|
+
if (!chainOrder)
|
|
949
|
+
return 'unknown';
|
|
950
|
+
if (!chainOrderUnchangedFromCache(chainOrder, cachedRaw))
|
|
951
|
+
return 'unknown';
|
|
952
|
+
return 'absent';
|
|
953
|
+
}
|
|
721
954
|
/**
|
|
722
955
|
* Execute operations with retry on BroadcastUncertainError.
|
|
956
|
+
*
|
|
957
|
+
* Never re-broadcasts blindly: an uncertain broadcast may have landed, and
|
|
958
|
+
* re-sending the same ops would duplicate on-chain orders. A retry is only
|
|
959
|
+
* allowed on AUTHORITATIVE ABSENCE — a successful non-empty, non-truncated
|
|
960
|
+
* chain read where every op verifies 'absent' (see verifyOpAgainstChain). An
|
|
961
|
+
* empty read (node may be lagging), a truncated read (get_full_accounts
|
|
962
|
+
* capped the result set; fresh creates sort last and are the first entries
|
|
963
|
+
* omitted), or any 'landed'/'unknown' verdict defers to the post-broadcast
|
|
964
|
+
* reconciliation machinery (pollChainForConfirmation +
|
|
965
|
+
* reconcileAfterUncertainBroadcast), which verifies inclusion and adopts
|
|
966
|
+
* landed orders before the next cycle.
|
|
723
967
|
* @param {import('./dexbot_class').DEXBot} bot
|
|
724
968
|
* @param {Array} operations
|
|
725
969
|
* @param {Array} opContexts
|
|
@@ -736,26 +980,93 @@ async function executeWithRetryOnUncertain(bot, operations, opContexts) {
|
|
|
736
980
|
&& !err.partialOnChainState
|
|
737
981
|
&& attempt <= MAX_RETRIES;
|
|
738
982
|
if (isRetriable) {
|
|
739
|
-
|
|
740
|
-
|
|
983
|
+
// Verify per operation kind against a live snapshot before
|
|
984
|
+
// re-broadcasting (see verifyOpAgainstChain): only a provably
|
|
985
|
+
// unapplied batch may be retried; a truncated or empty read is
|
|
986
|
+
// never authoritative (nodes lag / get_full_accounts caps the
|
|
987
|
+
// window) → defer.
|
|
988
|
+
let absence = 'unknown';
|
|
741
989
|
try {
|
|
742
990
|
const accountRef = bot.accountId || bot.account?.id || bot.account;
|
|
743
|
-
const
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
991
|
+
const freshRead = await chainOrders.readOpenOrdersWithMeta(accountRef);
|
|
992
|
+
const freshChain = freshRead.orders;
|
|
993
|
+
if (Array.isArray(freshChain) && freshChain.length > 0) {
|
|
994
|
+
absence = 'absent';
|
|
995
|
+
for (const ctx of opContexts) {
|
|
996
|
+
if (!ctx)
|
|
997
|
+
continue;
|
|
998
|
+
const verdict = verifyOpAgainstChain(bot, freshChain, ctx);
|
|
999
|
+
if (verdict !== 'absent') {
|
|
1000
|
+
absence = verdict;
|
|
1001
|
+
break;
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
// A truncated read (get_full_accounts caps limit_orders, and
|
|
1006
|
+
// fresh creates sort last in the by_account index) omits the
|
|
1007
|
+
// very orders this batch may have landed — 'absent' is not
|
|
1008
|
+
// authoritative here, degrade to 'unknown' and defer.
|
|
1009
|
+
if (freshRead.truncated && absence === 'absent') {
|
|
1010
|
+
absence = 'unknown';
|
|
748
1011
|
}
|
|
749
1012
|
}
|
|
750
|
-
catch (
|
|
751
|
-
bot.manager.logger.log(`[COW] Pre-retry
|
|
1013
|
+
catch (verifyErr) {
|
|
1014
|
+
bot.manager.logger.log(`[COW] Pre-retry chain verification failed (non-fatal): ${verifyErr?.message || verifyErr}`, 'warn');
|
|
752
1015
|
}
|
|
753
|
-
|
|
1016
|
+
if (absence === 'absent') {
|
|
1017
|
+
bot.manager.logger.log(`[COW] Broadcast uncertain (attempt ${attempt}/${MAX_RETRIES + 1}); verified unapplied on chain, retrying...`, 'warn');
|
|
1018
|
+
// The CREATE prep recorded pending-broadcast entries for this
|
|
1019
|
+
// batch; re-entering executeOperationsWithStrategy would hit the
|
|
1020
|
+
// PENDING_BROADCASTS guard and fall back to structural resync
|
|
1021
|
+
// instead of re-broadcasting. We have just PROVEN this batch is
|
|
1022
|
+
// absent on chain (authoritative read), so dropping this batch's
|
|
1023
|
+
// own entries is safe and lets the intended re-broadcast happen.
|
|
1024
|
+
// Only THIS batch's slots are dropped; entries from other
|
|
1025
|
+
// unresolved batches are preserved (their broadcasts may have
|
|
1026
|
+
// landed, so clearing them could re-create duplicates).
|
|
1027
|
+
const retriedCreateSlots = (opContexts || [])
|
|
1028
|
+
.filter((ctx) => ctx && ctx.kind === 'create')
|
|
1029
|
+
.map((ctx) => ({ type: COW_ACTIONS.CREATE, id: ctx.id }));
|
|
1030
|
+
if (retriedCreateSlots.length > 0) {
|
|
1031
|
+
clearPendingBroadcastsForSlots(bot, retriedCreateSlots);
|
|
1032
|
+
}
|
|
1033
|
+
await bot._ensureCredentialDaemonWritable('COW batch retry');
|
|
1034
|
+
continue;
|
|
1035
|
+
}
|
|
1036
|
+
bot.manager.logger.log(`[COW] Broadcast uncertain (attempt ${attempt}/${MAX_RETRIES + 1}); ` +
|
|
1037
|
+
`${absence === 'landed' ? 'operation(s) confirmed applied on chain' : 'chain state unverifiable (empty/truncated/lagging read)'} — ` +
|
|
1038
|
+
`deferring to post-broadcast reconciliation (no blind re-broadcast)`, 'warn');
|
|
1039
|
+
throw err;
|
|
754
1040
|
}
|
|
755
1041
|
throw err;
|
|
756
1042
|
}
|
|
757
1043
|
}
|
|
758
1044
|
}
|
|
1045
|
+
/**
|
|
1046
|
+
* Whether a chain order still matches the cached pre-update state the
|
|
1047
|
+
* limit_order_update delta was built from. Only a provably-unchanged order
|
|
1048
|
+
* makes a re-broadcast of the identical delta safe (it applies to the same
|
|
1049
|
+
* base). Any other state (target applied, filled, resized) must defer.
|
|
1050
|
+
* @param {Object} chainOrder - Raw chain order object (get_full_accounts)
|
|
1051
|
+
* @param {Object|null} cachedRaw - The rawOnChain cache captured at build time
|
|
1052
|
+
* @returns {boolean}
|
|
1053
|
+
*/
|
|
1054
|
+
function chainOrderUnchangedFromCache(chainOrder, cachedRaw) {
|
|
1055
|
+
if (!chainOrder || !cachedRaw)
|
|
1056
|
+
return false;
|
|
1057
|
+
const base = chainOrder.sell_price?.base;
|
|
1058
|
+
const quote = chainOrder.sell_price?.quote;
|
|
1059
|
+
const cachedBase = cachedRaw.sell_price?.base?.amount;
|
|
1060
|
+
const cachedQuote = cachedRaw.sell_price?.quote?.amount;
|
|
1061
|
+
const cachedForSale = cachedRaw.for_sale;
|
|
1062
|
+
if (base === undefined || quote === undefined)
|
|
1063
|
+
return false;
|
|
1064
|
+
if (cachedForSale === undefined || cachedBase === undefined || cachedQuote === undefined)
|
|
1065
|
+
return false;
|
|
1066
|
+
return String(base.amount ?? '') === String(cachedBase)
|
|
1067
|
+
&& String(quote.amount ?? '') === String(cachedQuote)
|
|
1068
|
+
&& String(chainOrder.for_sale ?? '') === String(cachedForSale);
|
|
1069
|
+
}
|
|
759
1070
|
/**
|
|
760
1071
|
* Execute blockchain operations with appropriate strategy (single batch or pair mode).
|
|
761
1072
|
* @param {import('./dexbot_class').DEXBot} bot
|
|
@@ -1050,7 +1361,7 @@ function buildCowResultFromPlan(bot, plan) {
|
|
|
1050
1361
|
...action.order,
|
|
1051
1362
|
id: action.id,
|
|
1052
1363
|
state: ORDER_STATES.VIRTUAL,
|
|
1053
|
-
orderId: null
|
|
1364
|
+
orderId: null,
|
|
1054
1365
|
});
|
|
1055
1366
|
}
|
|
1056
1367
|
else if (action.type === COW_ACTIONS.UPDATE) {
|
|
@@ -1074,9 +1385,8 @@ function buildCowResultFromPlan(bot, plan) {
|
|
|
1074
1385
|
size: rotatedSize,
|
|
1075
1386
|
price: rotatedPrice,
|
|
1076
1387
|
state: ORDER_STATES.VIRTUAL,
|
|
1077
|
-
orderId: null
|
|
1388
|
+
orderId: null,
|
|
1078
1389
|
});
|
|
1079
|
-
continue;
|
|
1080
1390
|
}
|
|
1081
1391
|
const current = workingGrid.get(action.id);
|
|
1082
1392
|
if (!current)
|
|
@@ -1089,7 +1399,7 @@ function buildCowResultFromPlan(bot, plan) {
|
|
|
1089
1399
|
...(action.order || {}),
|
|
1090
1400
|
id: action.id,
|
|
1091
1401
|
orderId: action.orderId || current.orderId,
|
|
1092
|
-
size: newSize
|
|
1402
|
+
size: newSize,
|
|
1093
1403
|
});
|
|
1094
1404
|
}
|
|
1095
1405
|
}
|
|
@@ -1133,7 +1443,7 @@ function applyRotationTransitionsToWorkingGrid(bot, workingGrid, executedContext
|
|
|
1133
1443
|
...sourceSlot,
|
|
1134
1444
|
state: ORDER_STATES.VIRTUAL,
|
|
1135
1445
|
orderId: null,
|
|
1136
|
-
rawOnChain: null
|
|
1446
|
+
rawOnChain: null,
|
|
1137
1447
|
});
|
|
1138
1448
|
bot.manager.logger.log(`[COW] Pre-applied rotation: source ${oldOrder.id} → VIRTUAL (order ${sourceSlot.orderId} moved to ${newGridId})`, 'debug');
|
|
1139
1449
|
}
|
|
@@ -1154,7 +1464,7 @@ function applyRotationTransitionsToWorkingGrid(bot, workingGrid, executedContext
|
|
|
1154
1464
|
// already held a prior committed ID (e.g. a partial commit
|
|
1155
1465
|
// left a stale reference). The source-of-truth is always the
|
|
1156
1466
|
// original order being rotated.
|
|
1157
|
-
orderId: oldOrder?.orderId || destSlot.orderId || null
|
|
1467
|
+
orderId: oldOrder?.orderId || destSlot.orderId || null,
|
|
1158
1468
|
});
|
|
1159
1469
|
bot.manager.logger.log(`[COW] Pre-applied rotation: dest ${newGridId} → ACTIVE (orderId=${oldOrder?.orderId || destSlot.orderId || 'none'})`, 'debug');
|
|
1160
1470
|
}
|
|
@@ -1190,7 +1500,18 @@ async function pollChainForConfirmation(bot, opContexts, options = {}) {
|
|
|
1190
1500
|
let remaining = [...createContexts];
|
|
1191
1501
|
for (let attempt = 1; attempt <= maxPollRetries; attempt++) {
|
|
1192
1502
|
try {
|
|
1193
|
-
const
|
|
1503
|
+
const chainRead = await readOpenOrdersWithMetaSafe(chainOrders, accountRef);
|
|
1504
|
+
const chainSnapshot = chainRead.orders;
|
|
1505
|
+
// A truncated read omits the freshest orders — the exact CREATEs
|
|
1506
|
+
// this poll is trying to confirm — so absence cannot be
|
|
1507
|
+
// distinguished from window truncation. Fall back to the
|
|
1508
|
+
// reconciliation machinery immediately instead of burning the
|
|
1509
|
+
// remaining polls.
|
|
1510
|
+
if (chainRead.truncated) {
|
|
1511
|
+
bot.manager.logger.log(`[COW][POLL] Chain read TRUNCATED (account exceeds the get_full_accounts window); ` +
|
|
1512
|
+
`fresh creates cannot be confirmed — deferring to reconciliation`, 'warn');
|
|
1513
|
+
break;
|
|
1514
|
+
}
|
|
1194
1515
|
if (!Array.isArray(chainSnapshot) || chainSnapshot.length === 0) {
|
|
1195
1516
|
if (attempt < maxPollRetries) {
|
|
1196
1517
|
await sleep(pollIntervalMs);
|
|
@@ -1255,13 +1576,129 @@ function restoreSkippedUpdateSlotsInWorkingGrid(bot, workingGrid, skippedSlotIds
|
|
|
1255
1576
|
}
|
|
1256
1577
|
bot.manager.logger.log(`[COW] Restored ${skippedSlotIds.size} slot(s) after ${skippedCount} skipped update action(s).`, 'debug');
|
|
1257
1578
|
}
|
|
1579
|
+
/**
|
|
1580
|
+
* Bounded re-plan for a stale pre-broadcast plan (regression-safe policy).
|
|
1581
|
+
*
|
|
1582
|
+
* Policy (bounded re-plan + proceed):
|
|
1583
|
+
* * First staleness hit → re-plan ONCE from fresh master using the same
|
|
1584
|
+
* fills; the recursion re-runs this guard against the fresh plan. A
|
|
1585
|
+
* re-plan with no executable actions means the grid is already consistent
|
|
1586
|
+
* post-fills — the stale plan must NOT ship.
|
|
1587
|
+
* * Still stale (master kept mutating), or no fill context to re-plan with
|
|
1588
|
+
* → PROCEED with the plan anyway and request a structural resync. Never
|
|
1589
|
+
* hard-abort on staleness: an abort would silently drop the fill set that
|
|
1590
|
+
* triggered this rebalance (_processFillsWithBatching only hard-aborts on
|
|
1591
|
+
* illegal-state or accounting failures), and the post-broadcast commit
|
|
1592
|
+
* guard + chain adoption below close any residual divergence.
|
|
1593
|
+
*
|
|
1594
|
+
* Stack discipline: the original plan's grid is popped before the fresh
|
|
1595
|
+
* re-plan pushes (LIFO order); when the re-plan fails/aborts, the original
|
|
1596
|
+
* grid is pushed back (marker restored) so the later commit/catch pop sites
|
|
1597
|
+
* release exactly the entry they were pushed with, instead of underflowing
|
|
1598
|
+
* or stealing a nested grid's entry.
|
|
1599
|
+
*
|
|
1600
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
1601
|
+
* @param {Object} cowResult - The stale plan result
|
|
1602
|
+
* @param {number} replanDepth - Recursion depth (0 = first attempt)
|
|
1603
|
+
* @param {Object} preBroadcastGuard - The failed evaluateCommit result
|
|
1604
|
+
* @returns {Promise<{handled: boolean, result?: Object}>} handled=true when the
|
|
1605
|
+
* batch was resolved by the re-plan (fresh plan executed, or stale plan
|
|
1606
|
+
* skipped as already-consistent); handled=false when the caller must proceed
|
|
1607
|
+
* with the original plan.
|
|
1608
|
+
*/
|
|
1609
|
+
async function replanStaleBatch(bot, cowResult, replanDepth, preBroadcastGuard) {
|
|
1610
|
+
const canReplan = replanDepth < STALE_PLAN_REPLAN_LIMIT
|
|
1611
|
+
&& Array.isArray(cowResult.fills) && cowResult.fills.length > 0;
|
|
1612
|
+
if (!canReplan) {
|
|
1613
|
+
bot.manager.logger.log(`[COW] Plan stale pre-broadcast (${preBroadcastGuard.reason}); ` +
|
|
1614
|
+
(replanDepth >= STALE_PLAN_REPLAN_LIMIT
|
|
1615
|
+
? 'still stale after re-plan — proceeding with plan (commit guard + chain adoption close divergence)'
|
|
1616
|
+
: 'no fill context for re-plan — proceeding with plan (commit guard + chain adoption close divergence)'), 'warn');
|
|
1617
|
+
await requestStructuralResync(bot, 'plan stale pre-broadcast (proceeding with plan)', { reason: preBroadcastGuard.reason });
|
|
1618
|
+
return { handled: false };
|
|
1619
|
+
}
|
|
1620
|
+
bot.manager.logger.log(`[COW] Plan stale pre-broadcast (${preBroadcastGuard.reason}); re-planning once from fresh master`, 'warn');
|
|
1621
|
+
// Abandon the original plan's working grid: it can no longer commit. Pop
|
|
1622
|
+
// it so the rebalance stack stays balanced — the fresh plan's grid (pushed
|
|
1623
|
+
// by performSafeRebalance below) is popped by the recursion's own
|
|
1624
|
+
// commit/cleanup path. Guarded on the push marker (plan-path calls never
|
|
1625
|
+
// pushed a grid); the marker is cleared so a later throw in this frame
|
|
1626
|
+
// (e.g. the recursion) cannot pop the entry a second time.
|
|
1627
|
+
const hadPushedGrid = cowResult?._workingGridPushed === true;
|
|
1628
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1629
|
+
let replanned = null;
|
|
1630
|
+
try {
|
|
1631
|
+
// Restore the boundary-shift budget consumed by the abandoned plan:
|
|
1632
|
+
// it was built from the same fills and never shipped, so the re-plan
|
|
1633
|
+
// must derive from the FULL batch budget — not the leftover. Without
|
|
1634
|
+
// the restore, each stale-plan re-plan spends the budget twice and
|
|
1635
|
+
// drifts conservative (boundary under-shift).
|
|
1636
|
+
if (bot.manager?._boundaryShiftBudgetBase != null) {
|
|
1637
|
+
bot.manager._boundaryShiftBudget = bot.manager._boundaryShiftBudgetBase;
|
|
1638
|
+
}
|
|
1639
|
+
if (typeof bot.manager.performSafeRebalance === 'function') {
|
|
1640
|
+
replanned = await bot.manager.performSafeRebalance(cowResult.fills, cowResult.excludeIds || new Set());
|
|
1641
|
+
}
|
|
1642
|
+
}
|
|
1643
|
+
catch (replanErr) {
|
|
1644
|
+
bot.manager.logger.log(`[COW] Re-plan failed: ${getErrorMessage(replanErr)}; proceeding with original plan`, 'warn');
|
|
1645
|
+
}
|
|
1646
|
+
if (replanned && !replanned.aborted) {
|
|
1647
|
+
if (hasExecutableActions(replanned)) {
|
|
1648
|
+
// The original plan's ops are abandoned with its working grid;
|
|
1649
|
+
// drop THEIR pending-broadcast entries only, or the recursion's
|
|
1650
|
+
// own pending-broadcast guard would reject the fresh plan's
|
|
1651
|
+
// CREATEs. Entries from an earlier unresolved batch are
|
|
1652
|
+
// deliberately KEPT: the entry guard only covers CREATE batches,
|
|
1653
|
+
// so a create-less batch can reach this path while earlier
|
|
1654
|
+
// entries are still live — clearing them here would let the fresh
|
|
1655
|
+
// plan re-create slots whose earlier broadcast may have landed
|
|
1656
|
+
// (duplicate orders). The recursion's guard will then abort +
|
|
1657
|
+
// reconcile instead.
|
|
1658
|
+
clearPendingBroadcastsForSlots(bot, cowResult.actions);
|
|
1659
|
+
return {
|
|
1660
|
+
handled: true,
|
|
1661
|
+
result: await updateOrdersOnChainBatchCOW(bot, replanned, {
|
|
1662
|
+
replanDepth: replanDepth + 1
|
|
1663
|
+
}),
|
|
1664
|
+
};
|
|
1665
|
+
}
|
|
1666
|
+
// Re-plan confirms the grid is already consistent post-fills; the
|
|
1667
|
+
// stale original plan must NOT ship. Pop the fresh plan's grid too
|
|
1668
|
+
// (it was never committed).
|
|
1669
|
+
popPushedWorkingGrid(bot, replanned);
|
|
1670
|
+
clearPendingBroadcastsForSlots(bot, cowResult.actions);
|
|
1671
|
+
bot.manager.logger.log('[COW] Re-plan produced no executable actions; grid is already consistent post-fills, skipping stale plan', 'info');
|
|
1672
|
+
return { handled: true, result: { executed: false, hadRotation: false, skippedStalePlan: true } };
|
|
1673
|
+
}
|
|
1674
|
+
// Re-plan failed or aborted — the original plan proceeds after all. Its
|
|
1675
|
+
// grid was popped above to keep the stack LIFO-balanced for the fresh
|
|
1676
|
+
// plan; push it back (marker restored) so the broadcast commit / catch
|
|
1677
|
+
// pop sites release exactly the entry they were pushed with, instead of
|
|
1678
|
+
// underflowing or stealing a nested grid's entry.
|
|
1679
|
+
if (hadPushedGrid && cowResult.workingGrid) {
|
|
1680
|
+
if (typeof bot.manager._pushWorkingGridRef === 'function') {
|
|
1681
|
+
bot.manager._pushWorkingGridRef(cowResult.workingGrid, cowResult);
|
|
1682
|
+
}
|
|
1683
|
+
else {
|
|
1684
|
+
bot.manager._currentWorkingGridStack?.push?.(cowResult.workingGrid);
|
|
1685
|
+
bot.manager._resetRebalanceStateToDepth?.();
|
|
1686
|
+
cowResult._workingGridPushed = true;
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
bot.manager.logger.log('[COW] Re-plan unavailable; proceeding with original plan (commit guard + chain adoption close divergence)', 'warn');
|
|
1690
|
+
await requestStructuralResync(bot, 're-plan unavailable (proceeding with original plan)', { reason: preBroadcastGuard.reason });
|
|
1691
|
+
return { handled: false };
|
|
1692
|
+
}
|
|
1258
1693
|
/**
|
|
1259
1694
|
* COW broadcast: Execute blockchain operations and commit working grid on success.
|
|
1260
1695
|
* @param {import('./dexbot_class').DEXBot} bot
|
|
1261
1696
|
* @param {Object} cowResult
|
|
1697
|
+
* @param {Object} [options={}] - Internal execution options (replanDepth)
|
|
1262
1698
|
* @returns {Promise<Object>}
|
|
1263
1699
|
*/
|
|
1264
|
-
async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
1700
|
+
async function updateOrdersOnChainBatchCOW(bot, cowResult, options = {}) {
|
|
1701
|
+
const replanDepth = Number.isFinite(Number(options?.replanDepth)) ? Number(options.replanDepth) : 0;
|
|
1265
1702
|
bot._currentCycleId = (Number.isFinite(Number(bot._currentCycleId)) ? Number(bot._currentCycleId) : 0) + 1;
|
|
1266
1703
|
const { workingGrid, workingIndexes, workingBoundary, actions } = cowResult;
|
|
1267
1704
|
if (bot.config.dryRun) {
|
|
@@ -1274,6 +1711,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1274
1711
|
bot.manager.logger.log(`Dry run: would place ${createCount} new orders`, 'info');
|
|
1275
1712
|
if (updateCount > 0)
|
|
1276
1713
|
bot.manager.logger.log(`Dry run: would update ${updateCount} orders`, 'info');
|
|
1714
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1277
1715
|
return { executed: true, hadRotation: false };
|
|
1278
1716
|
}
|
|
1279
1717
|
const chainOrderCandidates = Array.isArray(bot.manager?._lastUnmatchedChainOrders)
|
|
@@ -1309,6 +1747,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1309
1747
|
// (valid CREATEs, CANCELs, UPDATEs) still proceeds.
|
|
1310
1748
|
const hasHardOccupiedViolation = createSlotValidation.violations.some((v) => v.reason === 'slot_occupied');
|
|
1311
1749
|
if (hasHardOccupiedViolation) {
|
|
1750
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1312
1751
|
return {
|
|
1313
1752
|
executed: false,
|
|
1314
1753
|
aborted: true,
|
|
@@ -1332,6 +1771,10 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1332
1771
|
`${actions.length} action(s) remaining in batch`, 'warn');
|
|
1333
1772
|
if (!actions.some((action) => action.type === COW_ACTIONS.CREATE)) {
|
|
1334
1773
|
if (actions.length === 0) {
|
|
1774
|
+
// Same exactly-once marker discipline as the other early
|
|
1775
|
+
// returns: a pushed working grid must be popped here or the
|
|
1776
|
+
// caller would leak the stack entry.
|
|
1777
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1335
1778
|
return { executed: false, hadRotation: false };
|
|
1336
1779
|
}
|
|
1337
1780
|
}
|
|
@@ -1342,6 +1785,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1342
1785
|
bot.manager.logger.log?.(`[RECOVERY-EXHAUSTED] Blocking ${actions.filter((a) => a.type === COW_ACTIONS.CREATE).length} CREATE(s) ` +
|
|
1343
1786
|
`(exhausted ${(exhaustedAge / 1000).toFixed(0)}s ago). ` +
|
|
1344
1787
|
`Waiting for next fill or sync cycle to reset recovery state.`, 'warn');
|
|
1788
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1345
1789
|
return {
|
|
1346
1790
|
executed: false,
|
|
1347
1791
|
aborted: true,
|
|
@@ -1376,6 +1820,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1376
1820
|
catch (recoverErr) {
|
|
1377
1821
|
bot.manager.logger.log(`[COW] Recovery from pending broadcasts failed: ${recoverErr?.message || recoverErr}`, 'error');
|
|
1378
1822
|
}
|
|
1823
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1379
1824
|
return {
|
|
1380
1825
|
executed: false,
|
|
1381
1826
|
aborted: true,
|
|
@@ -1392,10 +1837,24 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1392
1837
|
` — adopting via sync instead of cancelling`, 'info');
|
|
1393
1838
|
try {
|
|
1394
1839
|
const accountRef = bot.account;
|
|
1395
|
-
const
|
|
1396
|
-
|
|
1840
|
+
const freshRead = await chainOrders.readOpenOrdersWithMeta(accountRef);
|
|
1841
|
+
// Truncated-read guard: a partial get_full_accounts window omits the
|
|
1842
|
+
// freshest orders; syncing on it would virtualize live slots and
|
|
1843
|
+
// re-create duplicates. Defer the adoption to a clean read — the
|
|
1844
|
+
// unmatched orders keep blocking CREATEs until then.
|
|
1845
|
+
if (freshRead.truncated) {
|
|
1846
|
+
bot.manager.logger.log('[COW] Post-guard chain snapshot TRUNCATED; skipping adoption sync (partial snapshot would virtualize live slots) — unmatched chain orders keep blocking CREATEs', 'warn');
|
|
1847
|
+
}
|
|
1848
|
+
else if (freshRead.orders && freshRead.orders.length > 0) {
|
|
1849
|
+
const freshSnapshot = freshRead.orders;
|
|
1397
1850
|
const syncResult = await bot.manager.syncFromOpenOrders(freshSnapshot, {
|
|
1398
|
-
|
|
1851
|
+
// Accounting enabled: the adopted chain orders were never
|
|
1852
|
+
// registered in master (they are unmatched/orphan), so the
|
|
1853
|
+
// adoption must lock their capital. skipAccounting:true would
|
|
1854
|
+
// leave the optimistic balances drifted until the next fetch
|
|
1855
|
+
// — inconsistent with the open-orders loop convention
|
|
1856
|
+
// ('readOpenOrders' → skipAccounting:false).
|
|
1857
|
+
skipAccounting: false,
|
|
1399
1858
|
});
|
|
1400
1859
|
if (syncResult && Array.isArray(syncResult.unmatchedChainOrders)) {
|
|
1401
1860
|
const processed = (syncResult.filledOrders?.length || 0) +
|
|
@@ -1430,6 +1889,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1430
1889
|
await bot.manager.requestStructuralGridResync('unmatched chain orders before COW create', { unmatchedChainOrders: unmatchedChainOrders });
|
|
1431
1890
|
}
|
|
1432
1891
|
bot.manager.logger.log(`[COW] Rejecting CREATE batch after sync: working grid invalidated by master mutation`, 'info');
|
|
1892
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1433
1893
|
return {
|
|
1434
1894
|
executed: false,
|
|
1435
1895
|
aborted: true,
|
|
@@ -1673,16 +2133,41 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1673
2133
|
restoreSkippedUpdateSlotsInWorkingGrid(bot, workingGrid, skippedUpdateSlotIds, skippedUpdateCount);
|
|
1674
2134
|
}
|
|
1675
2135
|
if (operations.length === 0) {
|
|
1676
|
-
|
|
2136
|
+
// Pop the working grid: in the re-plan recursion the fresh plan's
|
|
2137
|
+
// grid was pushed by performSafeRebalance, and nothing downstream
|
|
2138
|
+
// will commit it — leaving it on the stack would stick the manager
|
|
2139
|
+
// in REBALANCING permanently (the outer frame already popped its
|
|
2140
|
+
// own grid before recursing). Guarded on the push marker so plan
|
|
2141
|
+
// path calls (never pushed) cannot pop an unrelated entry.
|
|
2142
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1677
2143
|
return { executed: false, hadRotation: false };
|
|
1678
2144
|
}
|
|
1679
2145
|
const validation = validateOperationFunds(bot, operations, assetA, assetB);
|
|
1680
2146
|
bot.manager.logger.log(validation.summary, validation.isValid ? 'info' : 'warn');
|
|
1681
2147
|
if (!validation.isValid) {
|
|
1682
2148
|
bot.manager.logger.log(`Skipping batch broadcast: ${validation.violations.length} fund violation(s) detected`, 'warn');
|
|
1683
|
-
bot
|
|
2149
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1684
2150
|
return { executed: false, hadRotation: false };
|
|
1685
2151
|
}
|
|
2152
|
+
// Refuse stale plans BEFORE broadcasting: a master-grid change during
|
|
2153
|
+
// planning (fills, syncs) makes the working grid invalid. Broadcasting
|
|
2154
|
+
// anyway would place orders the commit will refuse to register, leaving
|
|
2155
|
+
// on-chain state ahead of the grid.
|
|
2156
|
+
// NOTE: the commit-time evaluateCommit also rejects empty deltas; here
|
|
2157
|
+
// (pre-broadcast) that case is already covered by the operations.length
|
|
2158
|
+
// guard above, so only staleness and version-mismatch are checked.
|
|
2159
|
+
const preBroadcastGuard = evaluateCommit(workingGrid, {
|
|
2160
|
+
hasLock: false,
|
|
2161
|
+
currentVersion: bot.manager._gridVersion
|
|
2162
|
+
});
|
|
2163
|
+
if (!preBroadcastGuard.canCommit) {
|
|
2164
|
+
// Bounded re-plan + proceed — policy documented on replanStaleBatch.
|
|
2165
|
+
const replan = await replanStaleBatch(bot, cowResult, replanDepth, preBroadcastGuard);
|
|
2166
|
+
if (replan.handled) {
|
|
2167
|
+
return replan.result;
|
|
2168
|
+
}
|
|
2169
|
+
// Fall through: proceed with the current plan (bounded policy).
|
|
2170
|
+
}
|
|
1686
2171
|
await bot._ensureCredentialDaemonWritable('COW batch broadcast');
|
|
1687
2172
|
bot.manager.logger.log(`[COW] Broadcasting batch with ${operations.length} operations...`, 'info');
|
|
1688
2173
|
bot._lastBroadcastHeartbeatAt = Date.now();
|
|
@@ -1701,7 +2186,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1701
2186
|
.join(', ');
|
|
1702
2187
|
bot.manager.logger.log(`[COW] Refusing to commit working grid: ${missingCreateResults.length} CREATE op(s) ` +
|
|
1703
2188
|
`returned no chainOrderId (${missingSlots}). Discarding working grid and syncing from chain.`, 'error');
|
|
1704
|
-
bot
|
|
2189
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1705
2190
|
markMissingCreateResultsAsStructuralBlocker(bot, missingCreateResults);
|
|
1706
2191
|
await recoverAfterMissingCreateResults(bot, 'missing create operation results');
|
|
1707
2192
|
return {
|
|
@@ -1720,7 +2205,34 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1720
2205
|
// that depends on broadcast result data.
|
|
1721
2206
|
applyRotationTransitionsToWorkingGrid(bot, workingGrid, executedContexts);
|
|
1722
2207
|
bot.manager.logger.log('[COW] Blockchain success - committing working grid to master', 'info');
|
|
1723
|
-
|
|
2208
|
+
// _commitWorkingGrid releases the stack entry on every settle
|
|
2209
|
+
// path (return or throw) and clears the push marker via
|
|
2210
|
+
// options.result, so a later throw in this frame (e.g.
|
|
2211
|
+
// processBatchResults after a successful commit) cannot pop a
|
|
2212
|
+
// second time for the same grid in the batch catch below.
|
|
2213
|
+
const commitOk = await bot.manager._commitWorkingGrid(workingGrid, workingIndexes, workingBoundary, { skipRecalc: true, result: cowResult });
|
|
2214
|
+
if (!commitOk) {
|
|
2215
|
+
// Master changed during broadcast (e.g. a fill landed and was
|
|
2216
|
+
// processed concurrently) so the commit was refused. The batch
|
|
2217
|
+
// is on chain; adopt the placed orders from the chain so master
|
|
2218
|
+
// converges instead of remaining divergent until a later sync.
|
|
2219
|
+
bot.manager.logger.log('[COW] Commit refused after broadcast; adopting placed orders from chain to keep master in sync', 'warn');
|
|
2220
|
+
const adopted = await adoptPlacedBatchFromChain(bot, chainOrders, '[COW]');
|
|
2221
|
+
if (!adopted) {
|
|
2222
|
+
// Chain state unknown (empty/lagging read or sync failure):
|
|
2223
|
+
// keep the pending-broadcast protection so a later plan
|
|
2224
|
+
// cannot duplicate the placed orders, and let the structural
|
|
2225
|
+
// resync adopt them from the chain.
|
|
2226
|
+
bot.manager.logger.log('[COW] Commit refused and chain adoption unavailable; keeping pending-broadcast protection pending structural resync', 'error');
|
|
2227
|
+
await requestStructuralResync(bot, 'commit refused after broadcast (chain adoption unavailable)', { reason: 'chain-adoption-unavailable' });
|
|
2228
|
+
return { executed: false, hadRotation: false, commitRefused: true, chainAdoptionPending: true };
|
|
2229
|
+
}
|
|
2230
|
+
// Deduct create fees for the placed orders (mirrors
|
|
2231
|
+
// processBatchResults, which the refused path bypasses).
|
|
2232
|
+
await applyAdoptionFeeAccounting(bot, executedContexts);
|
|
2233
|
+
await persistGridAndClearPendingBroadcasts(bot, '[COW]');
|
|
2234
|
+
return { executed: false, hadRotation: false, commitRefused: true };
|
|
2235
|
+
}
|
|
1724
2236
|
const batchResult = await processBatchResults(bot, result, executedContexts);
|
|
1725
2237
|
const persistResult = await bot.manager.persistGrid();
|
|
1726
2238
|
if (persistResult && (persistResult.skipped || persistResult.isValid === false)) {
|
|
@@ -1747,13 +2259,12 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1747
2259
|
delete bot.manager._persistenceWarning;
|
|
1748
2260
|
}
|
|
1749
2261
|
bot._metrics.batchesExecuted++;
|
|
1750
|
-
bot.manager._clearWorkingGridRef();
|
|
1751
2262
|
clearPendingBroadcasts(bot.manager?._pendingBroadcasts);
|
|
1752
2263
|
return { ...batchResult, executed: true, hadRotation: true };
|
|
1753
2264
|
}
|
|
1754
2265
|
else {
|
|
1755
2266
|
bot.manager.logger.log('[COW] Blockchain failed - working grid discarded, master unchanged', 'warn');
|
|
1756
|
-
bot
|
|
2267
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1757
2268
|
clearPendingBroadcasts(bot.manager?._pendingBroadcasts);
|
|
1758
2269
|
return { ...result, executed: false, hadRotation: false };
|
|
1759
2270
|
}
|
|
@@ -1799,21 +2310,46 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1799
2310
|
bot.manager.logger.log(`[COW][UNCERTAIN] Chain polling confirmed all ${createCount} CREATE(s) on chain. Committing working grid directly.`, 'info');
|
|
1800
2311
|
}
|
|
1801
2312
|
applyRotationTransitionsToWorkingGrid(bot, workingGrid, opContexts);
|
|
1802
|
-
|
|
1803
|
-
//
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
2313
|
+
// Same exactly-once discipline as the success path:
|
|
2314
|
+
// _commitWorkingGrid pops on every settle path and clears
|
|
2315
|
+
// the push marker via options.result, so a later throw here
|
|
2316
|
+
// must not pop again in the batch catch below.
|
|
2317
|
+
const pollCommitOk = await bot.manager._commitWorkingGrid(workingGrid, workingIndexes, workingBoundary, { skipRecalc: true, result: cowResult });
|
|
2318
|
+
if (!pollCommitOk) {
|
|
2319
|
+
// Master moved while polling — same recovery as the
|
|
2320
|
+
// refused-commit path: adopt from chain, keep pending
|
|
2321
|
+
// protection if adoption is unavailable.
|
|
2322
|
+
bot.manager.logger.log(`[COW][UNCERTAIN] Poll-confirmed commit refused; adopting placed orders from chain`, 'warn');
|
|
2323
|
+
const pollAdopted = await adoptPlacedBatchFromChain(bot, chainOrders, '[COW][UNCERTAIN]');
|
|
2324
|
+
if (!pollAdopted) {
|
|
2325
|
+
bot.manager.logger.log('[COW][UNCERTAIN] Poll-refused commit with unavailable chain adoption; keeping pending protection pending structural resync', 'error');
|
|
2326
|
+
await requestStructuralResync(bot, 'poll-confirmed commit refused (chain adoption unavailable)', { reason: 'chain-adoption-unavailable' });
|
|
2327
|
+
return { executed: false, hadRotation: false, commitRefused: true, chainAdoptionPending: true };
|
|
1809
2328
|
}
|
|
2329
|
+
await applyAdoptionFeeAccounting(bot, opContexts);
|
|
2330
|
+
await persistGridAndClearPendingBroadcasts(bot, '[COW][UNCERTAIN]');
|
|
2331
|
+
return { executed: false, hadRotation: false, commitRefused: true, uncertainResolved: true };
|
|
1810
2332
|
}
|
|
1811
|
-
|
|
1812
|
-
|
|
2333
|
+
// Enrich master grid with chain-assigned order IDs and amounts;
|
|
2334
|
+
// accounting enabled so the adopted orders' capital is locked
|
|
2335
|
+
// and any cancelled orders release theirs. The adoption result
|
|
2336
|
+
// is authoritative: a truncated read right after the confirming
|
|
2337
|
+
// poll can omit the batch's own fresh creates (they sort last
|
|
2338
|
+
// in the by_account index), so clearing the pending protection
|
|
2339
|
+
// on a failed adoption would let the next cycle re-create the
|
|
2340
|
+
// VIRTUAL slots as duplicate on-chain orders. Keep the
|
|
2341
|
+
// protection and defer to a structural resync instead.
|
|
2342
|
+
const pollAdoptedOk = await adoptPlacedBatchFromChain(bot, chainOrders, '[COW][UNCERTAIN]');
|
|
2343
|
+
if (!pollAdoptedOk) {
|
|
2344
|
+
bot.manager.logger.log('[COW][UNCERTAIN] Poll-confirmed commit with unavailable chain adoption; keeping pending protection pending structural resync', 'error');
|
|
2345
|
+
await requestStructuralResync(bot, 'poll-confirmed commit (chain adoption unavailable)', { reason: 'chain-adoption-unavailable' });
|
|
2346
|
+
return { executed: false, hadRotation: false, commitRefused: false, chainAdoptionPending: true };
|
|
1813
2347
|
}
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
2348
|
+
// The commit happened without processBatchResults (no success
|
|
2349
|
+
// result to extract); deduct create fees so the optimistic
|
|
2350
|
+
// balance reflects the on-chain cost.
|
|
2351
|
+
await applyAdoptionFeeAccounting(bot, opContexts);
|
|
2352
|
+
await persistGridAndClearPendingBroadcasts(bot, '[COW][UNCERTAIN]');
|
|
1817
2353
|
return { executed: true, hadRotation: false, uncertainResolved: true };
|
|
1818
2354
|
}
|
|
1819
2355
|
}
|
|
@@ -1821,7 +2357,7 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1821
2357
|
bot.manager.logger.log(`[COW][UNCERTAIN] Chain polling threw unexpectedly: ${getErrorMessage(pollErr)}. Falling back to reconciliation.`, 'error');
|
|
1822
2358
|
}
|
|
1823
2359
|
}
|
|
1824
|
-
bot
|
|
2360
|
+
popPushedWorkingGrid(bot, cowResult);
|
|
1825
2361
|
if (err instanceof BroadcastUncertainError) {
|
|
1826
2362
|
return await reconcileAfterUncertainBroadcast(bot, err, opContexts);
|
|
1827
2363
|
}
|
|
@@ -1857,6 +2393,152 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1857
2393
|
}
|
|
1858
2394
|
}
|
|
1859
2395
|
}
|
|
2396
|
+
/**
|
|
2397
|
+
* Request a structural grid resync with the recovery-state flag raised, so a
|
|
2398
|
+
* later plan cannot duplicate orders placed by a batch whose chain adoption is
|
|
2399
|
+
* pending. No-op when the manager has no structural-resync handler.
|
|
2400
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
2401
|
+
* @param {string} reason - Human-readable resync reason
|
|
2402
|
+
* @param {Object} [details={}] - Details passed to the resync handler
|
|
2403
|
+
*/
|
|
2404
|
+
async function requestStructuralResync(bot, reason, details = {}) {
|
|
2405
|
+
if (typeof bot.manager?.requestStructuralGridResync !== 'function')
|
|
2406
|
+
return;
|
|
2407
|
+
if (bot.manager._recoveryState) {
|
|
2408
|
+
bot.manager._recoveryState = { ...bot.manager._recoveryState, structuralResyncRequested: true };
|
|
2409
|
+
}
|
|
2410
|
+
await bot.manager.requestStructuralGridResync(reason, details);
|
|
2411
|
+
}
|
|
2412
|
+
/**
|
|
2413
|
+
* Adopt a batch's placed orders from the chain after the commit was refused
|
|
2414
|
+
* or after a poll-confirmed uncertain commit. The batch ops are already on
|
|
2415
|
+
* chain but never reached master, so a full chain sync with accounting
|
|
2416
|
+
* enabled locks the placed orders' capital and releases any cancelled ones.
|
|
2417
|
+
* Returns true when the adoption sync ran; false when the chain state could
|
|
2418
|
+
* not be read (empty/lagging read, truncated read, or sync failure) — the
|
|
2419
|
+
* caller then keeps the pending-broadcast protection and defers adoption to
|
|
2420
|
+
* a structural resync.
|
|
2421
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
2422
|
+
* @param {Object} chainOrders - Chain orders module
|
|
2423
|
+
* @param {string} logPrefix - Log prefix for sync failure messages
|
|
2424
|
+
* @returns {Promise<boolean>}
|
|
2425
|
+
*/
|
|
2426
|
+
async function adoptPlacedBatchFromChain(bot, chainOrders, logPrefix) {
|
|
2427
|
+
try {
|
|
2428
|
+
const accountRef = bot.accountId || bot.account?.id || bot.account;
|
|
2429
|
+
const freshRead = await readOpenOrdersWithMetaSafe(chainOrders, accountRef);
|
|
2430
|
+
// A truncated read (get_full_accounts caps the limit_orders window and
|
|
2431
|
+
// fresh creates sort last) omits the very orders this batch just
|
|
2432
|
+
// broadcast — the adoption sync could not register them, and clearing
|
|
2433
|
+
// the pending-broadcast protection would let the next cycle re-create
|
|
2434
|
+
// them as duplicates on chain. Treat truncated like an unreadable
|
|
2435
|
+
// chain state: keep the protection and defer to a structural resync.
|
|
2436
|
+
if (freshRead.truncated) {
|
|
2437
|
+
bot.manager.logger.log(`${logPrefix} Chain read TRUNCATED after batch broadcast; adoption deferred (pending-broadcast protection kept)`, 'warn');
|
|
2438
|
+
return false;
|
|
2439
|
+
}
|
|
2440
|
+
const freshChain = freshRead.orders;
|
|
2441
|
+
if (freshChain.length > 0 && typeof bot.manager.syncFromOpenOrders === 'function') {
|
|
2442
|
+
await bot.manager.syncFromOpenOrders(freshChain, { skipAccounting: false });
|
|
2443
|
+
return true;
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
catch (syncErr) {
|
|
2447
|
+
bot.manager.logger.log(`${logPrefix} Chain sync after batch broadcast failed: ${getErrorMessage(syncErr)}`, 'error');
|
|
2448
|
+
}
|
|
2449
|
+
return false;
|
|
2450
|
+
}
|
|
2451
|
+
/**
|
|
2452
|
+
* Persist the master grid after a chain adoption and clear the pending
|
|
2453
|
+
* broadcast protection. Persist failures are logged, not thrown — the
|
|
2454
|
+
* in-memory master is authoritative and the next sync/persist converges disk.
|
|
2455
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
2456
|
+
* @param {string} logPrefix - Log prefix for persist failure messages
|
|
2457
|
+
*/
|
|
2458
|
+
async function persistGridAndClearPendingBroadcasts(bot, logPrefix) {
|
|
2459
|
+
try {
|
|
2460
|
+
await bot.manager.persistGrid();
|
|
2461
|
+
}
|
|
2462
|
+
catch (persistErr) {
|
|
2463
|
+
bot.manager.logger.log(`${logPrefix} Persist after chain adoption failed: ${getErrorMessage(persistErr)}`, 'error');
|
|
2464
|
+
}
|
|
2465
|
+
clearPendingBroadcasts(bot.manager?._pendingBroadcasts);
|
|
2466
|
+
}
|
|
2467
|
+
/**
|
|
2468
|
+
* Apply BTS create-fee accounting for a batch that bypassed the normal
|
|
2469
|
+
* processBatchResults pipeline (commit refused after broadcast, or
|
|
2470
|
+
* poll-confirmed uncertain commit). Mirrors the create branch of
|
|
2471
|
+
* processBatchResults using master-grid state after chain adoption, so the
|
|
2472
|
+
* optimistic balance reflects the on-chain create cost.
|
|
2473
|
+
*
|
|
2474
|
+
* Safe on adopted/committed slots only: the slot must already carry its
|
|
2475
|
+
* orderId, so the transition old(ACTIVE)→new(ACTIVE) is delta-zero and only
|
|
2476
|
+
* the fee is applied — no double capital commitment. Cancel/rotation fee
|
|
2477
|
+
* accounting is intentionally skipped: the chain sync already performed the
|
|
2478
|
+
* capital release for cancelled orders (diff-based, applying it again would
|
|
2479
|
+
* double-release), and rotations without a committed destination cannot be
|
|
2480
|
+
* accounted locally.
|
|
2481
|
+
* @param {import('./dexbot_class').DEXBot} bot
|
|
2482
|
+
* @param {Array<Object>} contexts - Executed op contexts (create/rotation/cancel)
|
|
2483
|
+
*/
|
|
2484
|
+
async function applyAdoptionFeeAccounting(bot, contexts) {
|
|
2485
|
+
if (!bot.manager?.accountant || !Array.isArray(contexts) || contexts.length === 0)
|
|
2486
|
+
return;
|
|
2487
|
+
const btsFeeData = getAssetFeesSafe('BTS');
|
|
2488
|
+
const btsSide = (typeof bot.manager.accountant._getBtsOrderType === 'function')
|
|
2489
|
+
? bot.manager.accountant._getBtsOrderType()
|
|
2490
|
+
: null;
|
|
2491
|
+
for (const ctx of contexts) {
|
|
2492
|
+
if (ctx?.kind === 'create') {
|
|
2493
|
+
const slot = ctx.order?.id ? bot.manager.orders.get(ctx.order.id) : null;
|
|
2494
|
+
if (!slot?.orderId)
|
|
2495
|
+
continue;
|
|
2496
|
+
try {
|
|
2497
|
+
await bot.manager.synchronizeWithChain({
|
|
2498
|
+
gridOrderId: ctx.order.id,
|
|
2499
|
+
chainOrderId: slot.orderId,
|
|
2500
|
+
isPartialPlacement: false,
|
|
2501
|
+
expectedType: ctx.order.type,
|
|
2502
|
+
fee: btsFeeData?.createFee || 0,
|
|
2503
|
+
}, 'createOrder');
|
|
2504
|
+
}
|
|
2505
|
+
catch (feeErr) {
|
|
2506
|
+
bot.manager.logger.log(`[COW] Adoption fee accounting failed for create slot ${ctx.order.id}: ${getErrorMessage(feeErr)}`, 'warn');
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
else if (ctx?.kind === 'cancel') {
|
|
2510
|
+
// The cancel landed but the commit was refused / the adoption path
|
|
2511
|
+
// bypassed processBatchResults. Master still holds the slot; the
|
|
2512
|
+
// next sync's phantom cleanup releases its commitment with fee 0 —
|
|
2513
|
+
// charge the cancel fee here so the optimistic BTS balance reflects
|
|
2514
|
+
// the on-chain cost exactly once (mirrors the sync's
|
|
2515
|
+
// 'cancel-order-unmatched-fee' pattern; the deferred-fee refund is
|
|
2516
|
+
// reconciled by the next sync's fill/cancel processing).
|
|
2517
|
+
if (btsSide && btsFeeData?.cancelFee > 0) {
|
|
2518
|
+
try {
|
|
2519
|
+
await bot.manager.accountant.adjustTotalBalance(btsSide, -btsFeeData.cancelFee, 'cancel-adopt-fee');
|
|
2520
|
+
}
|
|
2521
|
+
catch (feeErr) {
|
|
2522
|
+
bot.manager.logger.log(`[COW] Adoption fee accounting failed for cancel ${ctx.order?.orderId}: ${getErrorMessage(feeErr)}`, 'warn');
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
else if (ctx?.kind === 'size-update' || ctx?.kind === 'rotation') {
|
|
2527
|
+
// Same reasoning as cancels: the update landed but its fee was never
|
|
2528
|
+
// charged on this path; the next sync's size reconciliation applies
|
|
2529
|
+
// the chain state without an update fee (fee 0), so charge it once
|
|
2530
|
+
// here to prevent optimistic BTS drift.
|
|
2531
|
+
if (btsSide && btsFeeData?.updateFee > 0) {
|
|
2532
|
+
try {
|
|
2533
|
+
await bot.manager.accountant.adjustTotalBalance(btsSide, -btsFeeData.updateFee, 'update-adopt-fee');
|
|
2534
|
+
}
|
|
2535
|
+
catch (feeErr) {
|
|
2536
|
+
bot.manager.logger.log(`[COW] Adoption fee accounting failed for update ${ctx.kind === 'size-update' ? ctx.updateInfo?.partialOrder?.orderId : ctx.rotation?.oldOrder?.orderId}: ${getErrorMessage(feeErr)}`, 'warn');
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
}
|
|
2540
|
+
}
|
|
2541
|
+
}
|
|
1860
2542
|
/**
|
|
1861
2543
|
* Process results from batch transaction execution.
|
|
1862
2544
|
* Updates order state, synchronizes with chain, and deducts BTS fees.
|
|
@@ -1867,7 +2549,13 @@ async function updateOrdersOnChainBatchCOW(bot, cowResult) {
|
|
|
1867
2549
|
*/
|
|
1868
2550
|
async function processBatchResults(bot, result, opContexts) {
|
|
1869
2551
|
const results = extractOperationResults(result, 'processBatchResults', bot.manager?.logger?.log?.bind(bot.manager?.logger));
|
|
1870
|
-
|
|
2552
|
+
// Safe variant: this runs AFTER the working grid committed, so a throw
|
|
2553
|
+
// here (fee cache unset) would hard-fail the whole batch post-commit and
|
|
2554
|
+
// land in the executor catch, where the stack-entry marker is already
|
|
2555
|
+
// cleared — leaving the grid committed but its post-commit processing
|
|
2556
|
+
// (fee deduction, metadata) silently skipped. Zero-fee fallback keeps the
|
|
2557
|
+
// accounting close; the next sync converges the residual.
|
|
2558
|
+
const btsFeeData = getAssetFeesSafe('BTS');
|
|
1871
2559
|
let hadRotation = false;
|
|
1872
2560
|
let updateOperationCount = 0;
|
|
1873
2561
|
const updatesToApply = [];
|
|
@@ -1886,7 +2574,7 @@ async function processBatchResults(bot, result, opContexts) {
|
|
|
1886
2574
|
const oldOrder = ctx.updateInfo.partialOrder;
|
|
1887
2575
|
const ord = bot.manager.orders.get(oldOrder.id);
|
|
1888
2576
|
if (oldOrder && ord && bot.manager.accountant) {
|
|
1889
|
-
await bot.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData
|
|
2577
|
+
await bot.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData?.updateFee || 0, false);
|
|
1890
2578
|
}
|
|
1891
2579
|
if (ord) {
|
|
1892
2580
|
const updatedSlot = { ...ord, size: ctx.updateInfo.newSize };
|
|
@@ -1909,7 +2597,7 @@ async function processBatchResults(bot, result, opContexts) {
|
|
|
1909
2597
|
const chainOrderId = res && res[1];
|
|
1910
2598
|
if (chainOrderId) {
|
|
1911
2599
|
await bot.manager.synchronizeWithChain({
|
|
1912
|
-
gridOrderId: ctx.order.id, chainOrderId, expectedType: ctx.order.type, fee: btsFeeData
|
|
2600
|
+
gridOrderId: ctx.order.id, chainOrderId, expectedType: ctx.order.type, fee: btsFeeData?.createFee || 0
|
|
1913
2601
|
}, 'createOrder');
|
|
1914
2602
|
if (ctx.finalInts) {
|
|
1915
2603
|
const syncedOrder = bot.manager.orders.get(ctx.order.id);
|
|
@@ -1950,7 +2638,7 @@ async function processBatchResults(bot, result, opContexts) {
|
|
|
1950
2638
|
if (!newGridId) {
|
|
1951
2639
|
const ord = bot.manager.orders.get(oldOrder.id || rotation.id);
|
|
1952
2640
|
if (oldOrder && ord && bot.manager.accountant) {
|
|
1953
|
-
await bot.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData
|
|
2641
|
+
await bot.manager.accountant.updateOptimisticFreeBalance(oldOrder, ord, 'order-update', btsFeeData?.updateFee || 0, false);
|
|
1954
2642
|
}
|
|
1955
2643
|
if (ord) {
|
|
1956
2644
|
const updatedSlot = { ...ord, size: newSize };
|
|
@@ -2003,7 +2691,7 @@ async function processBatchResults(bot, result, opContexts) {
|
|
|
2003
2691
|
};
|
|
2004
2692
|
}
|
|
2005
2693
|
if (oldOrder && updatedSlot && bot.manager.accountant) {
|
|
2006
|
-
await bot.manager.accountant.updateOptimisticFreeBalance(oldOrder, updatedSlot, 'order-update', btsFeeData
|
|
2694
|
+
await bot.manager.accountant.updateOptimisticFreeBalance(oldOrder, updatedSlot, 'order-update', btsFeeData?.updateFee || 0, false);
|
|
2007
2695
|
}
|
|
2008
2696
|
if (oldOrder?.id && oldOrder.id !== newGridId) {
|
|
2009
2697
|
const currentSource = bot.manager.orders.get(oldOrder.id);
|
|
@@ -2042,6 +2730,8 @@ module.exports = {
|
|
|
2042
2730
|
formatUnmatchedChainOrderForLog,
|
|
2043
2731
|
recordPendingBroadcast,
|
|
2044
2732
|
clearPendingBroadcasts,
|
|
2733
|
+
clearPendingBroadcastsForSlots,
|
|
2734
|
+
popPushedWorkingGrid,
|
|
2045
2735
|
buildChainOrderFingerprint,
|
|
2046
2736
|
normalizeChainOrderForPendingMatch,
|
|
2047
2737
|
findChainOrderForSlot,
|