dexbot 1.7.0 → 1.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +3 -3
  3. package/analysis/README.md +33 -13
  4. package/claw/package.json +1 -1
  5. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  6. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  7. package/dist/analysis/account_resolver.js +5 -6
  8. package/dist/analysis/analyze_liquidity.js +15 -4
  9. package/dist/analysis/bot_key_utils.d.ts +44 -1
  10. package/dist/analysis/bot_key_utils.js +76 -7
  11. package/dist/analysis/fills_cache.js +2 -1
  12. package/dist/analysis/grid_correction_check.d.ts +1 -19
  13. package/dist/analysis/grid_correction_check.js +12 -30
  14. package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
  15. package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
  16. package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
  17. package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
  18. package/dist/analysis/liquidity/liquidity_view.js +11 -0
  19. package/dist/analysis/resolve_bot_accounts.js +7 -6
  20. package/dist/analysis/trade_profitability.js +2 -1
  21. package/dist/credential-daemon.js +13 -2
  22. package/dist/dexbot.js +4 -4
  23. package/dist/modules/account_bots.d.ts +12 -1
  24. package/dist/modules/account_bots.js +7 -5
  25. package/dist/modules/account_orders.d.ts +12 -0
  26. package/dist/modules/account_orders.js +30 -0
  27. package/dist/modules/bitshares-native/transport.js +24 -6
  28. package/dist/modules/bitshares_client.d.ts +24 -6
  29. package/dist/modules/bitshares_client.js +44 -7
  30. package/dist/modules/bot_settings.d.ts +2 -0
  31. package/dist/modules/constants.d.ts +28 -3
  32. package/dist/modules/constants.js +79 -10
  33. package/dist/modules/dexbot_class.d.ts +23 -6
  34. package/dist/modules/dexbot_class.js +44 -0
  35. package/dist/modules/dexbot_cow_runtime.d.ts +128 -8
  36. package/dist/modules/dexbot_cow_runtime.js +688 -188
  37. package/dist/modules/dexbot_fill_runtime.js +124 -35
  38. package/dist/modules/dexbot_maintenance_runtime.d.ts +1 -1
  39. package/dist/modules/dexbot_maintenance_runtime.js +259 -79
  40. package/dist/modules/dexbot_startup_runtime.js +32 -20
  41. package/dist/modules/dexbot_state_recovery.d.ts +33 -1
  42. package/dist/modules/dexbot_state_recovery.js +167 -5
  43. package/dist/modules/node_connect_policy.d.ts +8 -1
  44. package/dist/modules/node_connect_policy.js +18 -2
  45. package/dist/modules/node_flap_score.d.ts +87 -0
  46. package/dist/modules/node_flap_score.js +124 -0
  47. package/dist/modules/node_health_cache.d.ts +40 -3
  48. package/dist/modules/node_health_cache.js +66 -20
  49. package/dist/modules/node_manager.d.ts +101 -19
  50. package/dist/modules/node_manager.js +255 -65
  51. package/dist/modules/order/accounting.d.ts +9 -4
  52. package/dist/modules/order/accounting.js +131 -19
  53. package/dist/modules/order/async_lock.d.ts +7 -1
  54. package/dist/modules/order/async_lock.js +7 -1
  55. package/dist/modules/order/export.js +25 -18
  56. package/dist/modules/order/fill_ledger.d.ts +99 -0
  57. package/dist/modules/order/fill_ledger.js +103 -0
  58. package/dist/modules/order/format.d.ts +24 -10
  59. package/dist/modules/order/format.js +30 -10
  60. package/dist/modules/order/grid.d.ts +75 -18
  61. package/dist/modules/order/grid.js +289 -72
  62. package/dist/modules/order/grid_reconcile.d.ts +5 -1
  63. package/dist/modules/order/grid_reconcile.js +333 -210
  64. package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
  65. package/dist/modules/order/grid_reconcile_internal.js +223 -91
  66. package/dist/modules/order/index.d.ts +18 -5
  67. package/dist/modules/order/index.js +5 -5
  68. package/dist/modules/order/lifecycle.d.ts +353 -0
  69. package/dist/modules/order/lifecycle.js +1109 -0
  70. package/dist/modules/order/manager.d.ts +277 -20
  71. package/dist/modules/order/manager.js +680 -64
  72. package/dist/modules/order/placement_lock.d.ts +102 -0
  73. package/dist/modules/order/placement_lock.js +222 -0
  74. package/dist/modules/order/strategy.js +13 -1
  75. package/dist/modules/order/sync_engine.d.ts +74 -114
  76. package/dist/modules/order/sync_engine.js +254 -198
  77. package/dist/modules/order/utils/math.d.ts +29 -4
  78. package/dist/modules/order/utils/math.js +61 -4
  79. package/dist/modules/order/utils/order.d.ts +106 -37
  80. package/dist/modules/order/utils/order.js +256 -81
  81. package/dist/modules/order/utils/system.d.ts +99 -6
  82. package/dist/modules/order/utils/system.js +249 -35
  83. package/dist/modules/order/utils/validate.d.ts +7 -1
  84. package/dist/modules/order/utils/validate.js +48 -2
  85. package/dist/modules/outcome.d.ts +54 -0
  86. package/dist/modules/outcome.js +51 -0
  87. package/dist/modules/settings_merge.d.ts +4 -0
  88. package/dist/modules/settings_merge.js +9 -0
  89. package/dist/modules/timing_policy.d.ts +82 -0
  90. package/dist/modules/timing_policy.js +244 -0
  91. package/dist/modules/types.d.ts +149 -22
  92. package/dist/scripts/analyze-credit.js +54 -17
  93. package/dist/scripts/analyze-orders.d.ts +4 -2
  94. package/dist/scripts/analyze-orders.js +113 -34
  95. package/dist/scripts/chart_targets.js +11 -15
  96. package/dist/scripts/test-credit-renewal.js +12 -5
  97. package/docs/COW_INVARIANTS.md +71 -8
  98. package/docs/DEXBOT_COMPARISON.md +6 -6
  99. package/docs/EVOLUTION.md +10 -8
  100. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  101. package/docs/GRID_PRICE_INVARIANT.md +1 -1
  102. package/docs/GRID_RECONCILE.md +18 -13
  103. package/docs/LOGGING.md +25 -1
  104. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +183 -26
  105. package/docs/README.md +23 -2
  106. package/docs/WORKFLOW.md +9 -3
  107. package/docs/architecture.md +68 -13
  108. package/docs/developer_guide.md +25 -8
  109. package/modules/README.md +6 -0
  110. package/package.json +3 -3
  111. package/scripts/README.md +16 -0
  112. package/scripts/pdev.sh +14 -1
  113. package/scripts/pmain.sh +23 -8
  114. package/scripts/ptest.sh +7 -0
  115. package/tests/README.md +30 -1
@@ -1,12 +1,13 @@
1
1
  import { createRequire } from 'node:module';
2
2
  const require = createRequire(import.meta.url);
3
3
  import { _countActiveOnGrid, _cancelChainOrder, _recoverStartupSyncFailure, _refreshStartupUpdatePlans, _executeStartupUpdateBatch, _executeStartupSequentialUpdateFallback, _executePlannedStartupCreates, _reconcileStartupSide, } from './grid_reconcile_internal.js';
4
- import { ORDER_TYPES, ORDER_STATES, TIMING } from '../constants.js';
4
+ import { ORDER_TYPES, ORDER_STATES } from '../constants.js';
5
5
  import { readOpenOrdersGuarded } from '../chain_orders.js';
6
- import { getAssetFeesSafe, pricesOnSameLevel } from './utils/math.js';
7
- import { isOrderPlaced, parseChainOrder, isOrderOnChain, findLiveOrderOwnerByChainId, chainOrderMatchesSlotWithTolerance, duplicateOrphanLogInfo, resolveEdgeCount, reviveVirtualSlotFromChain, } from './utils/order.js';
6
+ import { getAssetFeesSafe, pricesOnSameLevel, resolveGapSlots } from './utils/math.js';
7
+ import { isOrderPlaced, parseChainOrder, isOrderOnChain, findLiveOrderOwnerByChainId, chainOrderMatchesSlotWithTolerance, duplicateOrphanLogInfo, resolveEdgeCount, reviveVirtualSlotFromChain, getLivePendingBroadcasts, } from './utils/order.js';
8
8
  import * as Format from './format.js';
9
9
  import { getErrorMessage } from '../utils/errors.js';
10
+ import { readSlotLifecycle } from './lifecycle.js';
10
11
  function _startupChainSnapshotSignature(orders, manager) {
11
12
  const signature = new Map();
12
13
  for (const raw of Array.isArray(orders) ? orders : []) {
@@ -32,7 +33,11 @@ function _startupCancelPlanStillCurrent(plan, manager, initialChainSignature, cu
32
33
  if (currentChainSignature.get(chainOrderId) !== initialChainSignature.get(chainOrderId))
33
34
  return false;
34
35
  if (plan.gridOrderId) {
35
- if (plan.boundaryIdx !== undefined && (plan.boundaryIdx !== manager?.boundaryIdx || plan.gapSlots !== manager?._gapSlots))
36
+ // Geometry identity is compared through the same resolver the PLAN used
37
+ // (resolveGapSlots), never the raw `_gapSlots` field: an unpublished field
38
+ // would compare `undefined` against the plan's derived band and skip every
39
+ // cancellation as "stale".
40
+ if (plan.boundaryIdx !== undefined && (plan.boundaryIdx !== manager?.boundaryIdx || plan.gapSlots !== resolveGapSlots(manager)))
36
41
  return false;
37
42
  const slot = manager?.orders instanceof Map ? manager.orders.get(plan.gridOrderId) : null;
38
43
  if (!slot || !isOrderOnChain(slot) || slot.orderId !== plan.chainOrderId)
@@ -157,7 +162,7 @@ export async function decideStartupGridAction({ persistedGrid, chainOpenOrders,
157
162
  * @param {Array<Object>} params.chainOpenOrders - On-chain open orders
158
163
  * @returns {Promise<Object|null>} Reconciliation result, or null when reconciliation completes without a summary object
159
164
  */
160
- export async function reconcileGridOrders({ manager, config, account, privateKey, chainOrders, chainOpenOrders, }) {
165
+ export async function reconcileGridOrders({ manager, config, account, privateKey, chainOrders, chainOpenOrders, options = {}, }) {
161
166
  // Parameter validation
162
167
  if (!manager || typeof manager.synchronizeWithChain !== 'function') {
163
168
  throw new Error('reconcileGridOrders: manager must be provided with synchronizeWithChain method');
@@ -181,207 +186,294 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
181
186
  }
182
187
  const logger = manager && manager.logger;
183
188
  const dryRun = !!(config && config.dryRun);
184
- const parsedChain = (chainOpenOrders || [])
185
- .map((co) => ({ chain: co, parsed: parseChainOrder(co, manager.assets) }))
186
- .filter((x) => x.parsed !== null);
189
+ // A maintenance/targeted reconcile must NOT broadcast while a structural
190
+ // rebuild owns placement: Phase 1 mutates the master and Phase 2 places
191
+ // orders, so running it inside the rebuild re-creates the duplicate class
192
+ // the rebuild's own reconcile is designed to avoid. Rebuild callers pass
193
+ // allowDuringRebuild (they ARE the rebuild and run update-first).
194
+ const rebuildInProgress = manager.isStructuralRebuildInProgress?.() === true;
195
+ if (rebuildInProgress && options.allowDuringRebuild !== true) {
196
+ logger?.log?.('Reconcile deferred: structural grid rebuild in progress; the rebuild owns placement and update-first repair', 'info');
197
+ return null;
198
+ }
199
+ // PHASE 1: In-memory reconciliation under lock — planning runs with no
200
+ // blockchain I/O INSIDE the lock. All cancellations are collected into
201
+ // plannedCancels and executed in Phase 2. This keeps the lock hierarchy
202
+ // clean (no _gridLock held across sync calls).
203
+ //
204
+ // The exclusive placement region is acquired BEFORE the chain snapshot is
205
+ // used: the caller read the chain before this call, and waiting up to 120s
206
+ // for placement lets another path broadcast in between. Planning from that
207
+ // stale view is the duplicate-class race. After the acquire we re-read the
208
+ // chain (guarded: non-truncated, non-empty) so Phase 1/2 work from a
209
+ // snapshot no older than the region. Held across Phase 1 + Phase 2 and
210
+ // released after stopBroadcasting in the finally below.
211
+ // TOKEN-LEVEL HOLDER: this function acquires and releases its own placement
212
+ // lease because it must span Phase 1 (planning) and Phase 2 (broadcasting)
213
+ // — a `withPlacement` callback body would invert the phase structure. Every
214
+ // OTHER chain-mutating path goes through `manager.withPlacement()`; keep it
215
+ // that way (see placement_lock.ts).
216
+ let placementLease = null;
217
+ let regionStarted = false;
218
+ const releaseRegion = () => {
219
+ if (regionStarted)
220
+ manager.stopBroadcasting?.();
221
+ placementLease?.release();
222
+ placementLease = null;
223
+ };
224
+ let chainSnapshot = Array.isArray(chainOpenOrders) ? chainOpenOrders : [];
187
225
  const activeCfg = (config && config.coreOrders) ? config.coreOrders : {};
188
226
  let targetBuy = Math.max(0, Number.isFinite(Number(activeCfg.buy)) ? Number(activeCfg.buy) : 1)
189
227
  + resolveEdgeCount(config, 'buy');
190
228
  let targetSell = Math.max(0, Number.isFinite(Number(activeCfg.sell)) ? Number(activeCfg.sell) : 1)
191
229
  + resolveEdgeCount(config, 'sell');
192
- const chainBuys = parsedChain.filter((x) => x.parsed.type === ORDER_TYPES.BUY).map((x) => x.chain);
193
- const chainSells = parsedChain.filter((x) => x.parsed.type === ORDER_TYPES.SELL).map((x) => x.chain);
194
- // PHASE 1: In-memory reconciliation under lock — pure planning, no blockchain I/O.
195
- // All cancellations are collected into plannedCancels and executed in Phase 2.
196
- // This keeps the lock hierarchy clean (no _gridLock held across sync calls).
197
- const phase1Result = await manager._gridLock.acquire(async () => {
198
- if (typeof manager._applyOrderUpdate !== 'function') {
199
- throw new Error('manager._applyOrderUpdate is required for startup reconciliation');
200
- }
201
- const applyUpdate = manager._applyOrderUpdate.bind(manager);
202
- const chainIds = new Set((Array.isArray(chainOpenOrders) ? chainOpenOrders : []).map(o => o?.id).filter((id) => Boolean(id)));
203
- const chainOrderById = new Map();
204
- for (const raw of (Array.isArray(chainOpenOrders) ? chainOpenOrders : [])) {
205
- if (raw?.id)
206
- chainOrderById.set(String(raw.id), raw);
207
- }
208
- for (const order of manager.orders.values()) {
209
- if (isOrderPlaced(order)) {
210
- if (!order.orderId || !chainIds.has(order.orderId)) {
211
- // Absence-decision guard: only virtualize as a phantom when
212
- // the absence is trustworthy. An orderId assigned within the
213
- // sync-lock window may be an in-flight create/adopt whose
214
- // broadcast has not landed or is not yet visible to a
215
- // lagging/truncated read. Virtualizing it here and
216
- // re-creating would duplicate a real live order (the
217
- // reconcile-timeout death-spiral root cause). Mirror the
218
- // sync_engine committed-order guard; ghost orders (PARTIAL +
219
- // size=0) still pass through so known fills get cleaned up.
220
- const assignedAt = manager._orderIdAssignedAt?.get(order.orderId ?? '') || 0;
221
- const isGhost = order.size <= 0 && order.state === ORDER_STATES.PARTIAL;
222
- if (!isGhost && assignedAt > 0 && Date.now() - assignedAt < TIMING.SYNC_LOCK_TIMEOUT_MS) {
223
- logger?.log?.(`Startup: Order ${order.id} (ID ${order.orderId}) absent from snapshot but freshly assigned ` +
224
- `(${Date.now() - assignedAt}ms ago); deferring phantom cleanup (duplicate-order protection).`, 'warn');
225
- continue;
226
- }
227
- logger?.log?.(`Startup: Found phantom order ${order.id} (ID ${order.orderId}) not on chain. Resetting to VIRTUAL.`, 'warn');
228
- await applyUpdate({
229
- ...order,
230
- state: ORDER_STATES.VIRTUAL,
231
- orderId: "",
232
- rawOnChain: null
233
- }, 'startup-phantom', { skipAccounting: true });
234
- }
235
- }
236
- else if (order.orderId && chainIds.has(String(order.orderId))) {
237
- // DARK LEVEL: the slot is not placed but its chain order is
238
- // still live (a phantom cleanup on a truncated read, or a
239
- // refused-commit adoption). Left as-is it is invisible to
240
- // window accounting (_countActiveOnGrid counts only
241
- // ACTIVE/PARTIAL) while its order still counts in chainCount —
242
- // so this side reads a shortfall it can never repair, and the
243
- // planner sees a free level that already holds an order (the
244
- // duplicate-price precondition). Revive in place: no broadcast,
245
- // and only when the order really rests at this slot's level.
246
- const rawChain = chainOrderById.get(String(order.orderId)) ?? null;
247
- const parsedChain = rawChain ? parseChainOrder(rawChain, manager.assets) : null;
248
- // The helper logs the revive itself (one warn line, same text
249
- // on both call sites) — no second log here.
250
- await reviveVirtualSlotFromChain(manager, order, parsedChain, rawChain, 'startup-revive', { skipAccounting: true });
230
+ let phase1Result;
231
+ try {
232
+ if (typeof manager.acquirePlacement === 'function') {
233
+ placementLease = await manager.acquirePlacement('startup-reconcile', { timeoutMs: 120000 });
234
+ if (!placementLease) {
235
+ logger?.log?.('Startup reconcile: exclusive placement region unavailable (timeout/shutdown); deferring to the next sync cycle.', 'warn');
236
+ return null;
251
237
  }
252
238
  }
253
- const matchedChainOrderIds = new Set();
254
- for (const gridOrder of manager.orders.values()) {
255
- if (gridOrder && gridOrder.orderId) {
256
- matchedChainOrderIds.add(gridOrder.orderId);
239
+ try {
240
+ const refreshed = await readOpenOrdersGuarded(chainOrders, account, {
241
+ log: (msg, level) => logger?.log?.(msg, level),
242
+ label: 'STARTUP-RECONCILE',
243
+ detail: 'post-placement refresh',
244
+ deferEmpty: true,
245
+ });
246
+ if (Array.isArray(refreshed) && refreshed.length > 0) {
247
+ chainSnapshot = refreshed;
248
+ logger?.log?.(`Startup reconcile: refreshed ${chainSnapshot.length} open order(s) after acquiring placement`, 'debug');
257
249
  }
258
250
  }
259
- const unmatchedChain = (Array.isArray(chainOpenOrders) ? chainOpenOrders : []).filter((co) => co && !matchedChainOrderIds.has(co.id));
260
- let unmatchedParsed = unmatchedChain
251
+ catch (readErr) {
252
+ // Keep the caller's snapshot; the Phase 2 re-checks still guard the
253
+ // destructive decisions.
254
+ logger?.log?.(`Startup reconcile: post-acquire open-orders refresh failed (${getErrorMessage(readErr)}); using caller snapshot`, 'warn');
255
+ }
256
+ const parsedChain = chainSnapshot
261
257
  .map((co) => ({ chain: co, parsed: parseChainOrder(co, manager.assets) }))
262
258
  .filter((x) => x.parsed !== null);
263
- const plannedCancels = [];
264
- const cancelledDuplicateIds = new Set();
265
- const activeGridOrders = (Array.from(manager.orders.values())).filter((o) => o && o.orderId && isOrderPlaced(o));
266
- // Update-first policy: unmatched chain orders are NEVER cancelled here.
267
- // They flow into _reconcileStartupSide below, which price-updates them onto
268
- // rail slots in a single batch (plannedUpdates), creates missing orders, and
269
- // cancels only true surplus (chainCount > targetCount). Pre-emptive
270
- // cancellation would turn one batched update into per-order cancel+create
271
- // churn and strip the bot's own live orders during a reset.
272
- for (const u of unmatchedParsed) {
273
- const p = u.parsed;
274
- const desc = `Unmatched chain order: ${p.orderId} (${p.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL'}), price=${Format.formatPrice6(p.price)}, size=${Format.formatSizeByOrderType(p.size ?? 0, p.type, manager.assets)}`;
275
- let nearest = null;
276
- for (const gridOrder of activeGridOrders) {
277
- if (gridOrder.type !== p.type)
278
- continue;
279
- // Tolerance-based level check, NOT priceSlotEqual at the asset
280
- // precision (a low-precision asset collapses adjacent levels).
281
- const isEqual = pricesOnSameLevel(p.price, p.size ?? 0, p.type, gridOrder.price, gridOrder.size, gridOrder.type, manager.assets, manager.config?.incrementPercent);
282
- const priceDiff = Math.abs(p.price - gridOrder.price);
283
- const candidate = { gridOrder, priceDiff, isEqual };
284
- if (!nearest || priceDiff < nearest.priceDiff)
285
- nearest = candidate;
259
+ const chainBuys = parsedChain.filter((x) => x.parsed.type === ORDER_TYPES.BUY).map((x) => x.chain);
260
+ const chainSells = parsedChain.filter((x) => x.parsed.type === ORDER_TYPES.SELL).map((x) => x.chain);
261
+ // Start the broadcast region INSIDE the try so a throw releases the
262
+ // placement lease in the catch below.
263
+ regionStarted = typeof manager.startBroadcasting === 'function';
264
+ if (regionStarted)
265
+ manager.startBroadcasting?.();
266
+ phase1Result = await manager._gridLock.acquire(async () => {
267
+ if (typeof manager._applyOrderUpdate !== 'function') {
268
+ throw new Error('manager._applyOrderUpdate is required for startup reconciliation');
286
269
  }
287
- if (nearest && nearest.isEqual) {
288
- // The sync layer already owns this duplicate: PASS 2 queues a
289
- // cancel-only correction, and startup runs it (correctAllPriceMismatches)
290
- // before reconcile — leaving a stale snapshot where the order is already
291
- // cancelled. Defer to the correction so a single orphan is cancelled once,
292
- // but restore the untracked-fund accounting its cancel-only path does not
293
- // perform (reconcile's Phase-2 cancel used to do it via releaseUntrackedFunds).
294
- const alreadyQueued = Array.isArray(manager.ordersNeedingPriceCorrection) &&
295
- manager.ordersNeedingPriceCorrection.some((q) => q?.chainOrderId === p.orderId && q?.cancelOnly === true);
296
- const alreadyCancelled = typeof chainOrders.wasRecentlyOwnCancelled === 'function' &&
297
- chainOrders.wasRecentlyOwnCancelled(p.orderId);
298
- if (alreadyQueued || alreadyCancelled) {
299
- if (manager.accountant && manager._fundLock && p.size != null && p.size > 0) {
300
- const size = p.size;
301
- await manager._fundLock.acquire(async () => {
302
- await manager.accountant.addToChainFree(p.type, size, 'startup-skip-duplicate');
303
- });
270
+ const applyUpdate = manager._applyOrderUpdate.bind(manager);
271
+ const chainIds = new Set((Array.isArray(chainSnapshot) ? chainSnapshot : []).map(o => o?.id).filter((id) => Boolean(id)));
272
+ const chainOrderById = new Map();
273
+ for (const raw of (Array.isArray(chainSnapshot) ? chainSnapshot : [])) {
274
+ if (raw?.id)
275
+ chainOrderById.set(String(raw.id), raw);
276
+ }
277
+ for (const order of manager.orders.values()) {
278
+ if (isOrderPlaced(order)) {
279
+ if (!order.orderId || !chainIds.has(order.orderId)) {
280
+ // Absence-decision guard: only virtualize as a phantom when
281
+ // the absence is trustworthy. An orderId assigned within the
282
+ // sync-lock window may be an in-flight create/adopt whose
283
+ // broadcast has not landed or is not yet visible to a
284
+ // lagging/truncated read. Virtualizing it here and
285
+ // re-creating would duplicate a real live order (the
286
+ // reconcile-timeout death-spiral root cause). Same unified
287
+ // guard as the sync engine and the fund recalibration sweep,
288
+ // read through the slot so the persisted create-uncertain
289
+ // marker participates too. Ghost orders (PARTIAL + size=0)
290
+ // still pass through so known fills get cleaned up.
291
+ const lifecycle = readSlotLifecycle(manager, order);
292
+ const isGhost = order.size <= 0 && order.state === ORDER_STATES.PARTIAL;
293
+ if (!isGhost && lifecycle.isInFlight()) {
294
+ const reason = lifecycle.isRecentlyAssigned()
295
+ ? `freshly assigned ${Date.now() - lifecycle.assignedAt}ms ago`
296
+ : lifecycle.isUnresolved
297
+ ? 'its lifecycle phase is unresolved (a lost CREATE receipt or an ambiguous read)'
298
+ : 'recently committed';
299
+ logger?.log?.(`Startup: Order ${order.id} (ID ${order.orderId}) absent from snapshot but ${reason}; ` +
300
+ `deferring phantom cleanup (duplicate-order protection).`, 'warn');
301
+ continue;
302
+ }
303
+ logger?.log?.(`Startup: Found phantom order ${order.id} (ID ${order.orderId}) not on chain. Resetting to VIRTUAL.`, 'warn');
304
+ await applyUpdate({
305
+ ...order,
306
+ state: ORDER_STATES.VIRTUAL,
307
+ orderId: "",
308
+ rawOnChain: null
309
+ }, 'startup-phantom', { skipAccounting: true });
304
310
  }
305
- cancelledDuplicateIds.add(p.orderId);
306
- continue;
307
311
  }
308
- const { level, suffix } = duplicateOrphanLogInfo(p.orderId);
309
- logger?.log?.(`SUSPECTED DUPLICATE: ${desc} - nearest active grid ${nearest.gridOrder.id} ` +
310
- `(orderId=${nearest.gridOrder.orderId}, price=${Format.formatPrice6(nearest.gridOrder.price)}, ` +
311
- `diff=${Format.formatPrice6(nearest.priceDiff)}, slotEqual=${nearest.isEqual})${suffix}`, level);
312
- // Queue duplicate for Phase 2 cancellation instead of executing under lock
313
- cancelledDuplicateIds.add(p.orderId);
314
- plannedCancels.push({
315
- chainOrderId: p.orderId,
316
- chainOrderObj: u.chain,
317
- releaseUntrackedFunds: true,
318
- });
312
+ else if (order.orderId && chainIds.has(String(order.orderId))) {
313
+ // DARK LEVEL: the slot is not placed but its chain order is
314
+ // still live (a phantom cleanup on a truncated read, or a
315
+ // refused-commit adoption). Left as-is it is invisible to
316
+ // window accounting (_countActiveOnGrid counts only
317
+ // ACTIVE/PARTIAL) while its order still counts in chainCount —
318
+ // so this side reads a shortfall it can never repair, and the
319
+ // planner sees a free level that already holds an order (the
320
+ // duplicate-price precondition). Revive in place: no broadcast,
321
+ // and only when the order really rests at this slot's level.
322
+ const rawChain = chainOrderById.get(String(order.orderId)) ?? null;
323
+ const parsedChain = rawChain ? parseChainOrder(rawChain, manager.assets) : null;
324
+ // The helper logs the revive itself (one warn line, same text
325
+ // on both call sites) — no second log here.
326
+ await reviveVirtualSlotFromChain(manager, order, parsedChain, rawChain, 'startup-revive', { skipAccounting: true });
327
+ }
319
328
  }
320
- else if (nearest) {
321
- logger?.log?.(`${desc}; nearest active same-side grid ${nearest.gridOrder.id} ` +
322
- `(orderId=${nearest.gridOrder.orderId}, price=${Format.formatPrice6(nearest.gridOrder.price)}, ` +
323
- `diff=${Format.formatPrice6(nearest.priceDiff)}, slotEqual=${nearest.isEqual})`, 'warn');
329
+ const matchedChainOrderIds = new Set();
330
+ for (const gridOrder of manager.orders.values()) {
331
+ if (gridOrder && gridOrder.orderId) {
332
+ matchedChainOrderIds.add(gridOrder.orderId);
333
+ }
324
334
  }
325
- else {
326
- logger?.log?.(`${desc}; no active same-side grid order exists`, 'warn');
335
+ const unmatchedChain = (Array.isArray(chainSnapshot) ? chainSnapshot : []).filter((co) => co && !matchedChainOrderIds.has(co.id));
336
+ let unmatchedParsed = unmatchedChain
337
+ .map((co) => ({ chain: co, parsed: parseChainOrder(co, manager.assets) }))
338
+ .filter((x) => x.parsed !== null);
339
+ const plannedCancels = [];
340
+ const cancelledDuplicateIds = new Set();
341
+ const activeGridOrders = (Array.from(manager.orders.values())).filter((o) => o && o.orderId && isOrderPlaced(o));
342
+ // Update-first policy: unmatched chain orders are NEVER cancelled here.
343
+ // They flow into _reconcileStartupSide below, which price-updates them onto
344
+ // rail slots in a single batch (plannedUpdates), creates missing orders, and
345
+ // cancels only true surplus (chainCount > targetCount). Pre-emptive
346
+ // cancellation would turn one batched update into per-order cancel+create
347
+ // churn and strip the bot's own live orders during a reset.
348
+ for (const u of unmatchedParsed) {
349
+ const p = u.parsed;
350
+ const desc = `Unmatched chain order: ${p.orderId} (${p.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL'}), price=${Format.formatPrice6(p.price)}, size=${Format.formatSizeByOrderType(p.size ?? 0, p.type, manager.assets)}`;
351
+ let nearest = null;
352
+ for (const gridOrder of activeGridOrders) {
353
+ if (gridOrder.type !== p.type)
354
+ continue;
355
+ // Tolerance-based level check, NOT priceSlotEqual at the asset
356
+ // precision (a low-precision asset collapses adjacent levels).
357
+ const isEqual = pricesOnSameLevel(p.price, p.size ?? 0, p.type, gridOrder.price, gridOrder.size, gridOrder.type, manager.assets, manager.config?.incrementPercent);
358
+ const priceDiff = Math.abs(p.price - gridOrder.price);
359
+ const candidate = { gridOrder, priceDiff, isEqual };
360
+ if (!nearest || priceDiff < nearest.priceDiff)
361
+ nearest = candidate;
362
+ }
363
+ if (nearest && nearest.isEqual) {
364
+ // The sync layer already owns this duplicate: PASS 2 queues a
365
+ // cancel-only correction, and startup runs it (correctAllPriceMismatches)
366
+ // before reconcile — leaving a stale snapshot where the order is already
367
+ // cancelled. Defer to the correction so a single orphan is cancelled once,
368
+ // but restore the untracked-fund accounting its cancel-only path does not
369
+ // perform (reconcile's Phase-2 cancel used to do it via releaseUntrackedFunds).
370
+ const alreadyQueued = Array.isArray(manager.ordersNeedingPriceCorrection) &&
371
+ manager.ordersNeedingPriceCorrection.some((q) => q?.chainOrderId === p.orderId && q?.cancelOnly === true);
372
+ const alreadyCancelled = typeof chainOrders.wasRecentlyOwnCancelled === 'function' &&
373
+ chainOrders.wasRecentlyOwnCancelled(p.orderId);
374
+ if (alreadyQueued || alreadyCancelled) {
375
+ if (manager.accountant && manager._fundLock && p.size != null && p.size > 0) {
376
+ const size = p.size;
377
+ await manager._fundLock.acquire(async () => {
378
+ await manager.accountant.addToChainFree(p.type, size, 'startup-skip-duplicate');
379
+ });
380
+ }
381
+ cancelledDuplicateIds.add(p.orderId);
382
+ continue;
383
+ }
384
+ const { level, suffix } = duplicateOrphanLogInfo(p.orderId);
385
+ logger?.log?.(`SUSPECTED DUPLICATE: ${desc} - nearest active grid ${nearest.gridOrder.id} ` +
386
+ `(orderId=${nearest.gridOrder.orderId}, price=${Format.formatPrice6(nearest.gridOrder.price)}, ` +
387
+ `diff=${Format.formatPrice6(nearest.priceDiff)}, slotEqual=${nearest.isEqual})${suffix}`, level);
388
+ // Queue duplicate for Phase 2 cancellation instead of executing under lock
389
+ cancelledDuplicateIds.add(p.orderId);
390
+ plannedCancels.push({
391
+ chainOrderId: p.orderId,
392
+ chainOrderObj: u.chain,
393
+ releaseUntrackedFunds: true,
394
+ });
395
+ }
396
+ else if (nearest) {
397
+ logger?.log?.(`${desc}; nearest active same-side grid ${nearest.gridOrder.id} ` +
398
+ `(orderId=${nearest.gridOrder.orderId}, price=${Format.formatPrice6(nearest.gridOrder.price)}, ` +
399
+ `diff=${Format.formatPrice6(nearest.priceDiff)}, slotEqual=${nearest.isEqual})`, 'warn');
400
+ }
401
+ else {
402
+ logger?.log?.(`${desc}; no active same-side grid order exists`, 'warn');
403
+ }
327
404
  }
328
- }
329
- if (cancelledDuplicateIds.size > 0) {
330
- unmatchedParsed = unmatchedParsed.filter((u) => !cancelledDuplicateIds.has(u.parsed.orderId));
331
- }
332
- let unmatchedBuys = unmatchedParsed.filter((x) => x.parsed.type === ORDER_TYPES.BUY).map((x) => x.chain);
333
- let unmatchedSells = unmatchedParsed.filter((x) => x.parsed.type === ORDER_TYPES.SELL).map((x) => x.chain);
334
- const plannedCreates = [];
335
- const plannedUpdates = [];
336
- logger && logger.log && logger.log(`Startup reconcile starting: unmatched(sell=${unmatchedSells.length}, buy=${unmatchedBuys.length}), target(sell=${targetSell}, buy=${targetBuy})`, 'info');
337
- const sellResult = await _reconcileStartupSide({
338
- orderType: ORDER_TYPES.SELL,
339
- targetCount: targetSell,
340
- chainSideOrders: chainSells,
341
- unmatchedSideOrders: unmatchedSells,
342
- manager,
343
- chainOrders,
344
- account,
345
- privateKey,
346
- dryRun,
347
- plannedCreates,
348
- plannedUpdates,
349
- plannedCancels,
350
- planOnly: true,
351
- });
352
- const buyResult = await _reconcileStartupSide({
353
- orderType: ORDER_TYPES.BUY,
354
- targetCount: targetBuy,
355
- chainSideOrders: chainBuys,
356
- unmatchedSideOrders: unmatchedBuys,
357
- manager,
358
- chainOrders,
359
- account,
360
- privateKey,
361
- dryRun,
362
- plannedCreates,
363
- plannedUpdates,
364
- plannedCancels,
365
- planOnly: true,
405
+ if (cancelledDuplicateIds.size > 0) {
406
+ unmatchedParsed = unmatchedParsed.filter((u) => !cancelledDuplicateIds.has(u.parsed.orderId));
407
+ }
408
+ let unmatchedBuys = unmatchedParsed.filter((x) => x.parsed.type === ORDER_TYPES.BUY).map((x) => x.chain);
409
+ let unmatchedSells = unmatchedParsed.filter((x) => x.parsed.type === ORDER_TYPES.SELL).map((x) => x.chain);
410
+ const plannedCreates = [];
411
+ const plannedUpdates = [];
412
+ logger && logger.log && logger.log(`Startup reconcile starting: unmatched(sell=${unmatchedSells.length}, buy=${unmatchedBuys.length}), target(sell=${targetSell}, buy=${targetBuy})`, 'info');
413
+ const sellResult = await _reconcileStartupSide({
414
+ orderType: ORDER_TYPES.SELL,
415
+ targetCount: targetSell,
416
+ chainSideOrders: chainSells,
417
+ unmatchedSideOrders: unmatchedSells,
418
+ manager,
419
+ chainOrders,
420
+ account,
421
+ privateKey,
422
+ dryRun,
423
+ plannedCreates,
424
+ plannedUpdates,
425
+ plannedCancels,
426
+ planOnly: true,
427
+ });
428
+ const buyResult = await _reconcileStartupSide({
429
+ orderType: ORDER_TYPES.BUY,
430
+ targetCount: targetBuy,
431
+ chainSideOrders: chainBuys,
432
+ unmatchedSideOrders: unmatchedBuys,
433
+ manager,
434
+ chainOrders,
435
+ account,
436
+ privateKey,
437
+ dryRun,
438
+ plannedCreates,
439
+ plannedUpdates,
440
+ plannedCancels,
441
+ planOnly: true,
442
+ });
443
+ return { plannedCreates, plannedUpdates, plannedCancels, chainSellCount: sellResult.chainCount, chainBuyCount: buyResult.chainCount };
366
444
  });
367
- return { plannedCreates, plannedUpdates, plannedCancels, chainSellCount: sellResult.chainCount, chainBuyCount: buyResult.chainCount };
368
- });
445
+ }
446
+ catch (phase1Err) {
447
+ releaseRegion();
448
+ throw phase1Err;
449
+ }
369
450
  // PHASE 2: Blockchain operations outside lock (batch updates, creates, cancels, read)
370
451
  // These are the heavy operations that would block all other grid operations
371
452
  // if held under _gridLock. All lock acquisitions here follow the canonical
372
453
  // hierarchy (fillProcessingLock → syncLock → gridLock → fundLock).
373
- // P0 placement mutex: the whole of Phase 2 is broadcast I/O plus master-grid
374
- // mutations, and it is NOT under _gridLock. Without the broadcasting gate a
375
- // fill-driven COW rebalance can plan and broadcast the SAME slots in parallel
376
- // with this loop: the commit is then refused ("master mutation during
377
- // broadcasting"), both order generations land on chain, and the loser becomes
378
- // a duplicate-price orphan backlog that blocks placements until drained.
379
- // startBroadcasting() defers that rebalance —
380
- // manager.performSafeRebalance waits on the flag and the fill consumer gates
381
- // on it. The flag is refcounted and has a 120s stale-clear watchdog.
382
- if (typeof manager.startBroadcasting === 'function')
383
- manager.startBroadcasting();
454
+ //
455
+ // Placement was acquired before Phase 1, and the broadcast region was
456
+ // started there too, so a fill-driven COW rebalance cannot plan/broadcast
457
+ // the SAME slots in parallel with this loop. The refcount alone is
458
+ // ADVISORY (it only defers fills); the exclusive placement lease is the
459
+ // authoritative mutual exclusion and covers Phase 1's master mutations as
460
+ // well as Phase 2's I/O.
384
461
  try {
462
+ // LEASE-LIVENESS GATE (Phase 2 is the first mutation). This reconcile is
463
+ // one of the two token-level holders; the stale-region watchdog may have
464
+ // force-released the lease while Phase 1 or the pre-cancel chain read was
465
+ // slow, and another path can own the region by now. Phase 1 touched only
466
+ // the master (no chain effect), and the next cycle re-reads the chain
467
+ // after re-acquiring the region, so dropping this plan is recoverable —
468
+ // broadcasting it unprotected is the duplicate window the teardown was
469
+ // supposed to be safe about. The mirror of the COW body's check.
470
+ if (placementLease
471
+ && typeof manager.isPlacementLeaseCurrent === 'function'
472
+ && !manager.isPlacementLeaseCurrent(placementLease)) {
473
+ logger?.log?.('Startup reconcile: placement lease was force-released after planning (stale-region watchdog); ' +
474
+ 'discarding the plan instead of broadcasting without the region.', 'warn');
475
+ return null;
476
+ }
385
477
  const { plannedCreates, plannedUpdates, plannedCancels, chainSellCount, chainBuyCount } = phase1Result;
386
478
  // Execute planned cancellations (duplicates, edge, excess) outside lock.
387
479
  // Re-read immediately before the first cancellation: Phase 1 is pure
@@ -396,7 +488,7 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
396
488
  if (cancelPlan?.chainOrderId)
397
489
  phase2CancellationPlanIds.add(String(cancelPlan.chainOrderId));
398
490
  }
399
- const initialChainSignature = _startupChainSnapshotSignature(chainOpenOrders || [], manager);
491
+ const initialChainSignature = _startupChainSnapshotSignature(chainSnapshot || [], manager);
400
492
  let currentChainSignature = null;
401
493
  try {
402
494
  const currentChainOrders = await readOpenOrdersGuarded(chainOrders, account, {
@@ -609,35 +701,67 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
609
701
  const queuedCancellationIds = new Set((Array.isArray(manager.ordersNeedingPriceCorrection) ? manager.ordersNeedingPriceCorrection : [])
610
702
  .filter((entry) => entry?.chainOrderId && (entry?.cancelOnly === true || entry?.isSurplus === true))
611
703
  .map((entry) => String(entry.chainOrderId)));
704
+ // Absence/cancel decisions must never be taken on a snapshot
705
+ // that is lagging or that still has unresolved in-flight
706
+ // creates. Two independent guards:
707
+ // 1. every Phase-2 created ID must be visible — a missing
708
+ // one means this node has not caught up with our own
709
+ // broadcast;
710
+ // 2. no pending-broadcast entry may still be live — those
711
+ // are CREATEs from another (possibly concurrent) batch
712
+ // whose chain IDs are not known yet, so this pass cannot
713
+ // tell them apart from surplus and must not cancel.
714
+ // Skipping is safe: the next full sync/duplicate drain
715
+ // resolves the surplus with a caught-up view. Cancelling on
716
+ // the stale view is not (2026-11-03: 15
717
+ // legitimate fresh BUY orders this way).
718
+ const freshChainIds = new Set(freshParsed.map((x) => String(x.chain.id)));
719
+ const missingPhase2Ids = [...phase2CreatedOrderIds]
720
+ .filter((id) => !freshChainIds.has(String(id)));
721
+ const pendingBroadcastCount = getLivePendingBroadcasts(manager).length;
722
+ let skipSurplusSweepReason = '';
723
+ if (missingPhase2Ids.length > 0) {
724
+ skipSurplusSweepReason =
725
+ `post-Phase-2 chain read is lagging (${missingPhase2Ids.length} freshly created id(s) absent: ${missingPhase2Ids.slice(0, 5).join(', ')})`;
726
+ }
727
+ else if (pendingBroadcastCount > 0) {
728
+ skipSurplusSweepReason =
729
+ `${pendingBroadcastCount} pending-broadcast entr${pendingBroadcastCount === 1 ? 'y' : 'ies'} still protected (fresh CREATEs may be from another batch)`;
730
+ }
731
+ let cancelledSellCount = 0;
732
+ let cancelledBuyCount = 0;
612
733
  const staleSurplusCancels = [];
613
- for (const side of [ORDER_TYPES.SELL, ORDER_TYPES.BUY]) {
614
- const targetCount = side === ORDER_TYPES.SELL ? targetSell : targetBuy;
615
- let sideOrders = freshParsed.filter((x) => x.parsed.type === side);
616
- const sideLabel = side === ORDER_TYPES.SELL ? 'SELL' : 'BUY';
617
- if (sideOrders.length > targetCount) {
618
- // Sort by chain ID for deterministic cancellation order.
619
- sideOrders = sideOrders.sort((a, b) => (a.chain.id || '').localeCompare(b.chain.id || ''));
620
- let cancelLimit = sideOrders.length - targetCount;
621
- // Protected candidates do not consume cancelLimit;
622
- // selection therefore moves to the next eligible
623
- // sorted order instead of the pre-guard candidate.
624
- for (const so of sideOrders) {
625
- if (cancelLimit <= 0)
626
- break;
627
- const recentlyCancelled = typeof chainOrders.wasRecentlyOwnCancelled === 'function'
628
- && chainOrders.wasRecentlyOwnCancelled(so.chain.id);
629
- if (!gridOrderIds.has(so.chain.id)
630
- && !queuedCancellationIds.has(String(so.chain.id))
631
- && !phase2CancellationPlanIds.has(String(so.chain.id))
632
- && !recentlyCancelled) {
633
- staleSurplusCancels.push({ chainOrderObj: so.chain, sideLabel });
634
- cancelLimit--;
734
+ if (skipSurplusSweepReason) {
735
+ logger?.log?.(`Startup: skipping surplus sweep — ${skipSurplusSweepReason}`, 'warn');
736
+ }
737
+ else {
738
+ for (const side of [ORDER_TYPES.SELL, ORDER_TYPES.BUY]) {
739
+ const targetCount = side === ORDER_TYPES.SELL ? targetSell : targetBuy;
740
+ let sideOrders = freshParsed.filter((x) => x.parsed.type === side);
741
+ const sideLabel = side === ORDER_TYPES.SELL ? 'SELL' : 'BUY';
742
+ if (sideOrders.length > targetCount) {
743
+ // Sort by chain ID for deterministic cancellation order.
744
+ sideOrders = sideOrders.sort((a, b) => (a.chain.id || '').localeCompare(b.chain.id || ''));
745
+ let cancelLimit = sideOrders.length - targetCount;
746
+ // Protected candidates do not consume cancelLimit;
747
+ // selection therefore moves to the next eligible
748
+ // sorted order instead of the pre-guard candidate.
749
+ for (const so of sideOrders) {
750
+ if (cancelLimit <= 0)
751
+ break;
752
+ const recentlyCancelled = typeof chainOrders.wasRecentlyOwnCancelled === 'function'
753
+ && chainOrders.wasRecentlyOwnCancelled(so.chain.id);
754
+ if (!gridOrderIds.has(so.chain.id)
755
+ && !queuedCancellationIds.has(String(so.chain.id))
756
+ && !phase2CancellationPlanIds.has(String(so.chain.id))
757
+ && !recentlyCancelled) {
758
+ staleSurplusCancels.push({ chainOrderObj: so.chain, sideLabel });
759
+ cancelLimit--;
760
+ }
635
761
  }
636
762
  }
637
763
  }
638
764
  }
639
- let cancelledSellCount = 0;
640
- let cancelledBuyCount = 0;
641
765
  if (staleSurplusCancels.length > 0) {
642
766
  for (const sc of staleSurplusCancels) {
643
767
  try {
@@ -674,8 +798,7 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
674
798
  `gridActive(sell=${_countActiveOnGrid(manager, ORDER_TYPES.SELL)}, buy=${_countActiveOnGrid(manager, ORDER_TYPES.BUY)})`, 'info');
675
799
  }
676
800
  finally {
677
- if (typeof manager.stopBroadcasting === 'function')
678
- manager.stopBroadcasting();
801
+ releaseRegion();
679
802
  }
680
803
  return null;
681
804
  }