dexbot 1.4.6 → 1.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +1 -0
  2. package/dist/credential-daemon.d.ts.map +1 -1
  3. package/dist/credential-daemon.js +416 -92
  4. package/dist/credential-daemon.js.map +1 -1
  5. package/dist/dexbot.d.ts.map +1 -1
  6. package/dist/dexbot.js +61 -27
  7. package/dist/dexbot.js.map +1 -1
  8. package/dist/modules/account_bots.d.ts.map +1 -1
  9. package/dist/modules/account_bots.js +14 -24
  10. package/dist/modules/account_bots.js.map +1 -1
  11. package/dist/modules/bitshares-native/transport.d.ts.map +1 -1
  12. package/dist/modules/bitshares-native/transport.js +21 -0
  13. package/dist/modules/bitshares-native/transport.js.map +1 -1
  14. package/dist/modules/broadcast_failure.d.ts +21 -0
  15. package/dist/modules/broadcast_failure.d.ts.map +1 -0
  16. package/dist/modules/broadcast_failure.js +95 -0
  17. package/dist/modules/broadcast_failure.js.map +1 -0
  18. package/dist/modules/chain_keys.d.ts +2 -2
  19. package/dist/modules/chain_keys.d.ts.map +1 -1
  20. package/dist/modules/chain_keys.js +30 -56
  21. package/dist/modules/chain_keys.js.map +1 -1
  22. package/dist/modules/chain_orders.d.ts +79 -1
  23. package/dist/modules/chain_orders.d.ts.map +1 -1
  24. package/dist/modules/chain_orders.js +154 -10
  25. package/dist/modules/chain_orders.js.map +1 -1
  26. package/dist/modules/constants.d.ts +1 -0
  27. package/dist/modules/constants.d.ts.map +1 -1
  28. package/dist/modules/constants.js +21 -7
  29. package/dist/modules/constants.js.map +1 -1
  30. package/dist/modules/daemon_node_health.d.ts +33 -0
  31. package/dist/modules/daemon_node_health.d.ts.map +1 -0
  32. package/dist/modules/daemon_node_health.js +120 -0
  33. package/dist/modules/daemon_node_health.js.map +1 -0
  34. package/dist/modules/dexbot_class.d.ts +12 -3
  35. package/dist/modules/dexbot_class.d.ts.map +1 -1
  36. package/dist/modules/dexbot_class.js +19 -5
  37. package/dist/modules/dexbot_class.js.map +1 -1
  38. package/dist/modules/dexbot_cow_runtime.d.ts +46 -1
  39. package/dist/modules/dexbot_cow_runtime.d.ts.map +1 -1
  40. package/dist/modules/dexbot_cow_runtime.js +754 -64
  41. package/dist/modules/dexbot_cow_runtime.js.map +1 -1
  42. package/dist/modules/dexbot_credential_client.d.ts +15 -0
  43. package/dist/modules/dexbot_credential_client.d.ts.map +1 -1
  44. package/dist/modules/dexbot_credential_client.js +114 -141
  45. package/dist/modules/dexbot_credential_client.js.map +1 -1
  46. package/dist/modules/dexbot_fill_runtime.d.ts.map +1 -1
  47. package/dist/modules/dexbot_fill_runtime.js +45 -15
  48. package/dist/modules/dexbot_fill_runtime.js.map +1 -1
  49. package/dist/modules/dexbot_maintenance_runtime.d.ts +5 -1
  50. package/dist/modules/dexbot_maintenance_runtime.d.ts.map +1 -1
  51. package/dist/modules/dexbot_maintenance_runtime.js +238 -85
  52. package/dist/modules/dexbot_maintenance_runtime.js.map +1 -1
  53. package/dist/modules/dexbot_startup_runtime.d.ts.map +1 -1
  54. package/dist/modules/dexbot_startup_runtime.js +95 -33
  55. package/dist/modules/dexbot_startup_runtime.js.map +1 -1
  56. package/dist/modules/dexbot_state_recovery.d.ts.map +1 -1
  57. package/dist/modules/dexbot_state_recovery.js +24 -2
  58. package/dist/modules/dexbot_state_recovery.js.map +1 -1
  59. package/dist/modules/key_store.d.ts.map +1 -1
  60. package/dist/modules/key_store.js +5 -4
  61. package/dist/modules/key_store.js.map +1 -1
  62. package/dist/modules/launcher/credential_daemon.d.ts.map +1 -1
  63. package/dist/modules/launcher/credential_daemon.js +15 -3
  64. package/dist/modules/launcher/credential_daemon.js.map +1 -1
  65. package/dist/modules/node_failure_ledger.d.ts +39 -0
  66. package/dist/modules/node_failure_ledger.d.ts.map +1 -0
  67. package/dist/modules/node_failure_ledger.js +88 -0
  68. package/dist/modules/node_failure_ledger.js.map +1 -0
  69. package/dist/modules/node_health_cache.d.ts +2 -1
  70. package/dist/modules/node_health_cache.d.ts.map +1 -1
  71. package/dist/modules/node_health_cache.js +25 -0
  72. package/dist/modules/node_health_cache.js.map +1 -1
  73. package/dist/modules/node_manager.d.ts +3 -0
  74. package/dist/modules/node_manager.d.ts.map +1 -1
  75. package/dist/modules/node_manager.js +41 -10
  76. package/dist/modules/node_manager.js.map +1 -1
  77. package/dist/modules/order/accounting.d.ts +26 -20
  78. package/dist/modules/order/accounting.d.ts.map +1 -1
  79. package/dist/modules/order/accounting.js +281 -139
  80. package/dist/modules/order/accounting.js.map +1 -1
  81. package/dist/modules/order/grid.d.ts.map +1 -1
  82. package/dist/modules/order/grid.js +66 -14
  83. package/dist/modules/order/grid.js.map +1 -1
  84. package/dist/modules/order/grid_reconcile.d.ts.map +1 -1
  85. package/dist/modules/order/grid_reconcile.js +136 -56
  86. package/dist/modules/order/grid_reconcile.js.map +1 -1
  87. package/dist/modules/order/grid_reconcile_internal.d.ts +6 -5
  88. package/dist/modules/order/grid_reconcile_internal.d.ts.map +1 -1
  89. package/dist/modules/order/grid_reconcile_internal.js +388 -61
  90. package/dist/modules/order/grid_reconcile_internal.js.map +1 -1
  91. package/dist/modules/order/manager.d.ts +98 -4
  92. package/dist/modules/order/manager.d.ts.map +1 -1
  93. package/dist/modules/order/manager.js +379 -92
  94. package/dist/modules/order/manager.js.map +1 -1
  95. package/dist/modules/order/strategy.d.ts.map +1 -1
  96. package/dist/modules/order/strategy.js +24 -1
  97. package/dist/modules/order/strategy.js.map +1 -1
  98. package/dist/modules/order/sync_engine.d.ts +16 -3
  99. package/dist/modules/order/sync_engine.d.ts.map +1 -1
  100. package/dist/modules/order/sync_engine.js +341 -253
  101. package/dist/modules/order/sync_engine.js.map +1 -1
  102. package/dist/modules/order/utils/math.d.ts +79 -1
  103. package/dist/modules/order/utils/math.d.ts.map +1 -1
  104. package/dist/modules/order/utils/math.js +18 -0
  105. package/dist/modules/order/utils/math.js.map +1 -1
  106. package/dist/modules/order/utils/order.d.ts +44 -1
  107. package/dist/modules/order/utils/order.d.ts.map +1 -1
  108. package/dist/modules/order/utils/order.js +91 -5
  109. package/dist/modules/order/utils/order.js.map +1 -1
  110. package/dist/modules/order/utils/system.d.ts +5 -1
  111. package/dist/modules/order/utils/system.d.ts.map +1 -1
  112. package/dist/modules/order/utils/system.js +7 -5
  113. package/dist/modules/order/utils/system.js.map +1 -1
  114. package/dist/modules/order/utils/withPoolRef.d.ts.map +1 -1
  115. package/dist/modules/order/utils/withPoolRef.js +39 -4
  116. package/dist/modules/order/utils/withPoolRef.js.map +1 -1
  117. package/dist/modules/socket_json_client.d.ts +14 -0
  118. package/dist/modules/socket_json_client.d.ts.map +1 -0
  119. package/dist/modules/socket_json_client.js +100 -0
  120. package/dist/modules/socket_json_client.js.map +1 -0
  121. package/package.json +2 -1
  122. package/scripts/clear-all.sh +4 -4
  123. package/scripts/clear-logs.sh +4 -4
@@ -62,6 +62,7 @@ exports._buildOutsideInCreateGroups = _buildOutsideInCreateGroups;
62
62
  exports._executePlannedStartupCreates = _executePlannedStartupCreates;
63
63
  exports._reconcileStartupSide = _reconcileStartupSide;
64
64
  const constants_1 = require("../constants");
65
+ const chain_orders_1 = require("../chain_orders");
65
66
  const math_1 = require("./utils/math");
66
67
  const order_1 = require("./utils/order");
67
68
  const system_1 = require("./utils/system");
@@ -90,10 +91,41 @@ function _countActiveOnGrid(manager, type) {
90
91
  function _pickVirtualSlotsToActivate(manager, type, count) {
91
92
  if (count <= 0)
92
93
  return [];
94
+ // Boundary geometry: slots in the spread band (between boundary and
95
+ // sellStart) must never be activated as BUY/SELL — an order placed there
96
+ // would sit inside the gap and remove the spread. The SPREAD GUARD keeps
97
+ // such slots typed BUY/SELL (never SPREAD+ACTIVE), so a VIRTUAL sell left
98
+ // behind by a rotation/rebalance could otherwise be re-picked here and
99
+ // placed back inside the gap. Filter by geometry, not just type.
100
+ const gapSlots = (manager._gapSlots ?? calculateGapSlotsFromConfig(manager)) || 0;
101
+ const boundaryIdx = manager.boundaryIdx;
102
+ const sellStartIdx = (boundaryIdx !== null && Number.isFinite(Number(boundaryIdx)))
103
+ ? Number(boundaryIdx) + gapSlots + 1
104
+ : null;
105
+ const slotIndex = (id) => {
106
+ if (typeof id !== 'string')
107
+ return null;
108
+ const match = /^slot-(\d+)$/.exec(id);
109
+ if (!match)
110
+ return null;
111
+ const idx = parseInt(match[1], 10);
112
+ return Number.isFinite(idx) ? idx : null;
113
+ };
114
+ const inRail = (slot) => {
115
+ if (sellStartIdx === null)
116
+ return true;
117
+ const idx = slotIndex(slot.id);
118
+ if (idx === null)
119
+ return true;
120
+ if (type === constants_1.ORDER_TYPES.BUY)
121
+ return idx <= Number(boundaryIdx);
122
+ return idx >= sellStartIdx;
123
+ };
93
124
  // CRITICAL FIX: Filter by type BEFORE sorting
94
125
  // Only get slots of the requested type (SELL or BUY), not a mix
95
126
  const slotsOfType = Array.from(manager.orders.values())
96
127
  .filter((slot) => slot && slot.type === type)
128
+ .filter(inRail)
97
129
  .sort((a, b) => type === constants_1.ORDER_TYPES.BUY ? b.price - a.price : a.price - b.price);
98
130
  let effectiveMin = 0;
99
131
  try {
@@ -116,6 +148,14 @@ function _pickVirtualSlotsToActivate(manager, type, count) {
116
148
  }
117
149
  return valid;
118
150
  }
151
+ function calculateGapSlotsFromConfig(manager) {
152
+ try {
153
+ return (0, math_1.calculateGapSlots)(manager?.config?.incrementPercent, manager?.config?.targetSpreadPercent, manager?.config?.gridLimits);
154
+ }
155
+ catch (e) {
156
+ return null;
157
+ }
158
+ }
119
159
  function _getStartupSideComparators(orderType, assets) {
120
160
  const isSell = orderType === constants_1.ORDER_TYPES.SELL;
121
161
  const sortUpdateComparator = isSell
@@ -265,16 +305,16 @@ async function _cancelLargestOrder({ chainOrders, account, privateKey, manager,
265
305
  * @param {boolean} params.dryRun - Whether to simulate.
266
306
  * @returns {Promise<void>}
267
307
  */
268
- async function _createOrderFromGrid({ chainOrders, account, privateKey, manager, gridOrder, dryRun }) {
308
+ async function _createOrderFromGrid({ chainOrders, account, privateKey, manager, gridOrder, dryRun, extraOptions = {} }) {
269
309
  if (dryRun)
270
- return;
310
+ return null;
271
311
  // ATOMIC RE-VERIFICATION: Ensure slot is still virtual and hasn't been filled by recovery sync.
272
312
  // This protects the same slot (own orderId). The collision check below protects
273
313
  // against other slots at the same price — the two are complementary.
274
314
  const currentSlot = manager.orders.get(gridOrder.id);
275
315
  if (currentSlot && currentSlot.orderId) {
276
316
  manager.logger?.log?.(`[_createOrderFromGrid] SKIP: Slot ${gridOrder.id} already has orderId ${currentSlot.orderId}`, 'warn');
277
- return;
317
+ return null;
278
318
  }
279
319
  // Price collision guard: reject if another placed order already exists at this price level.
280
320
  const createPrice = gridOrder.price;
@@ -283,41 +323,64 @@ async function _createOrderFromGrid({ chainOrders, account, privateKey, manager,
283
323
  if (priceCollision) {
284
324
  manager.logger?.log?.(`[_createOrderFromGrid] SKIP: Create for ${gridOrder.id} at ${Format.formatPrice6(createPrice)} ` +
285
325
  `collides with placed order ${priceCollision.id} (${priceCollision.orderId}) at ${Format.formatPrice6(priceCollision.price)}`, 'warn');
286
- return;
326
+ return null;
287
327
  }
288
328
  const { amountToSell, sellAssetId, minToReceive, receiveAssetId } = (0, order_1.buildCreateOrderArgs)(gridOrder, manager.assets.assetA, manager.assets.assetB);
289
- const result = await chainOrders.createOrder(account, privateKey, amountToSell, sellAssetId, minToReceive, receiveAssetId, null, false);
329
+ const result = await chainOrders.createOrder(account, privateKey, amountToSell, sellAssetId, minToReceive, receiveAssetId, null, false, extraOptions);
290
330
  if (result && result.skipped) {
291
331
  const logger = manager && manager.logger;
292
332
  logger?.log?.(`[_createOrderFromGrid] Skipped slot ${gridOrder.id}: order amounts too small to place on-chain`, 'warn');
293
- return;
333
+ return null;
294
334
  }
295
335
  const operationResults = (0, order_1.extractBatchOperationResults)(result) || [];
296
336
  const chainOrderId = operationResults[0] && operationResults[0][1];
297
337
  if (chainOrderId) {
298
- const btsFeeData = (0, math_1.getAssetFees)('BTS');
299
- // Centralized Fund Tracking: Use manager's sync core to handle state transition and fund deduction
300
- await manager._applySync({
301
- gridOrderId: gridOrder.id,
302
- chainOrderId,
303
- isPartialPlacement: false,
304
- expectedType: gridOrder.type,
305
- fee: btsFeeData.createFee
306
- }, 'createOrder');
338
+ // Capture chain order ID BEFORE _applySync, so even if _applySync
339
+ // fails (leaving the order unregistered in manager.orders), Phase 3
340
+ // will still know this order was legitimately created and avoid cancelling it.
341
+ const capturedId = chainOrderId;
342
+ try {
343
+ const btsFeeData = (0, math_1.getAssetFees)('BTS');
344
+ await manager._applySync({
345
+ gridOrderId: gridOrder.id,
346
+ chainOrderId,
347
+ isPartialPlacement: false,
348
+ expectedType: gridOrder.type,
349
+ fee: btsFeeData.createFee
350
+ }, 'createOrder');
351
+ }
352
+ catch (syncErr) {
353
+ const logger = manager && manager.logger;
354
+ logger?.log?.(`[_createOrderFromGrid] _applySync failed after successful broadcast for ${capturedId}: ${(0, errors_1.getErrorMessage)(syncErr)}`, 'error');
355
+ // Run recovery sync to register the orphaned order. If this also
356
+ // fails (or the chain read is empty/truncated), the returned
357
+ // capturedId still prevents Phase 3 cancellation.
358
+ await _recoverSyncFromChain({
359
+ chainOrders,
360
+ manager,
361
+ account,
362
+ logger,
363
+ source: 'createOrder-applySync-failure',
364
+ });
365
+ // Return the captured ID regardless so Phase 3 knows this order was
366
+ // legitimately created and avoids cancelling it.
367
+ return capturedId;
368
+ }
369
+ return capturedId;
307
370
  }
308
371
  else {
309
372
  // CRITICAL FIX: Recovery sync if order extraction fails
310
373
  const logger = manager && manager.logger;
311
374
  logger?.log?.(`[_createOrderFromGrid] CRITICAL: createOrder succeeded but chainOrderId extraction failed`, 'error');
312
- try {
313
- const freshChainOrders = await chainOrders.readOpenOrders((0, system_1.resolveAccountRef)(manager, account), constants_1.TIMING.CONNECTION_TIMEOUT_MS);
314
- await manager.syncFromOpenOrders(freshChainOrders, {
315
- skipAccounting: false,
316
- source: 'chainOrderIdExtractionFailure',
317
- });
318
- }
319
- catch (syncErr) {
320
- logger?.log?.(`[_createOrderFromGrid] Recovery sync failed: ${(0, errors_1.getErrorMessage)(syncErr)}`, 'error');
375
+ const recovered = await _recoverSyncFromChain({
376
+ chainOrders,
377
+ manager,
378
+ account,
379
+ logger,
380
+ source: 'chainOrderIdExtractionFailure',
381
+ });
382
+ if (!recovered) {
383
+ logger?.log?.(`[_createOrderFromGrid] Recovery sync unavailable; zeroing slot ${gridOrder.id} to prevent duplicate creation`, 'error');
321
384
  // Transition the slot to zero-size VIRTUAL to prevent duplicate
322
385
  // creation on the next COW cycle. The orphaned chain order will
323
386
  // be picked up by the next normal full sync.
@@ -334,6 +397,7 @@ async function _createOrderFromGrid({ chainOrders, account, privateKey, manager,
334
397
  }
335
398
  }
336
399
  }
400
+ return null;
337
401
  }
338
402
  /**
339
403
  * Cancel a chain order and sync with manager.
@@ -356,11 +420,23 @@ async function _cancelChainOrder({ chainOrders, account, privateKey, manager, ch
356
420
  return;
357
421
  const cancelResult = await chainOrders.cancelOrder(account, privateKey, chainOrderId);
358
422
  if (cancelResult?.verifiedAfterFailure) {
359
- const freshChainOrders = await chainOrders.readOpenOrders((0, system_1.resolveAccountRef)(manager, account), constants_1.TIMING.CONNECTION_TIMEOUT_MS);
360
- await manager.syncFromOpenOrders(freshChainOrders, {
361
- skipAccounting: false,
423
+ const recovered = await _recoverSyncFromChain({
424
+ chainOrders,
425
+ manager,
426
+ account,
427
+ logger: manager && manager.logger,
362
428
  source: 'cancelOrder',
363
429
  });
430
+ if (!recovered) {
431
+ // The cancellation was authoritatively confirmed absent inside
432
+ // cancelOrder, but the recovery refetch was skipped (empty/
433
+ // truncated read — ambiguous, so the full sync must NOT run) or
434
+ // failed. The order is provably gone, so apply the local cancel
435
+ // sync to release its capital and clear the slot — same fallback
436
+ // as the dust-cancel path. Skipping it would leave the slot stuck
437
+ // ACTIVE with a chain order that no longer exists.
438
+ await manager._applySync({ orderId: chainOrderId }, 'cancelOrder');
439
+ }
364
440
  }
365
441
  else {
366
442
  await manager._applySync({ orderId: chainOrderId }, 'cancelOrder');
@@ -370,14 +446,37 @@ async function _cancelChainOrder({ chainOrders, account, privateKey, manager, ch
370
446
  if (releaseUntrackedFunds && manager.accountant && chainOrderObj) {
371
447
  const parsed = (0, order_1.parseChainOrder)(chainOrderObj, manager.assets);
372
448
  if (parsed && parsed.size != null && parsed.size > 0) {
373
- await manager.accountant.addToChainFree(parsed.type, parsed.size, 'startup-cancel-unmatched');
449
+ await manager._fundLock.acquire(async () => {
450
+ await manager.accountant.addToChainFree(parsed.type, parsed.size, 'startup-cancel-unmatched');
451
+ });
374
452
  }
375
453
  }
376
454
  }
377
- async function _recoverStartupSyncFailure({ chainOrders, manager, account, logger, triggerMessage, source }) {
455
+ /**
456
+ * Recovery sync from a fresh chain read, guarded against ambiguous snapshots.
457
+ * An empty read (node may be lagging behind a just-broadcast transaction) and
458
+ * a truncated read (get_full_accounts caps the limit_orders window; fresh
459
+ * orders sort last and are the first entries omitted) are both treated as
460
+ * unreadable: running syncFromOpenOrders on them would let pass-1 phantom
461
+ * cleanup virtualize ACTIVE/PARTIAL slots that are actually live on chain,
462
+ * after which the next cycle re-creates them and duplicates the orders.
463
+ * Returns the fresh orders when the sync ran, null when skipped/failed.
464
+ * @returns {Promise<any[] | null>}
465
+ * @private
466
+ */
467
+ async function _recoverSyncFromChain({ chainOrders, manager, account, logger, source, triggerMessage, skipMessage }) {
378
468
  try {
379
- logger?.log?.(triggerMessage, 'warn');
380
- const freshChainOrders = await chainOrders.readOpenOrders((0, system_1.resolveAccountRef)(manager, account), constants_1.TIMING.CONNECTION_TIMEOUT_MS);
469
+ if (triggerMessage)
470
+ logger?.log?.(triggerMessage, 'warn');
471
+ const freshChainOrders = await (0, chain_orders_1.readOpenOrdersGuarded)(chainOrders, (0, system_1.resolveAccountRef)(manager, account), {
472
+ log: (message, level) => logger?.log?.(message, level),
473
+ label: 'RECOVERY',
474
+ deferEmpty: true,
475
+ timeoutMs: constants_1.TIMING.CONNECTION_TIMEOUT_MS,
476
+ skipMessage: skipMessage || 'Recovery sync skipped - empty/truncated chain read is ambiguous (node may be lagging, or the get_full_accounts window omitted fresh orders); deferring to next reconcile cycle',
477
+ });
478
+ if (freshChainOrders === null)
479
+ return null;
381
480
  await manager.syncFromOpenOrders(freshChainOrders, {
382
481
  skipAccounting: false,
383
482
  source,
@@ -385,10 +484,22 @@ async function _recoverStartupSyncFailure({ chainOrders, manager, account, logge
385
484
  return freshChainOrders;
386
485
  }
387
486
  catch (syncErr) {
388
- logger?.log?.(`Startup: Recovery sync failed: ${(0, errors_1.getErrorMessage)(syncErr)}`, 'error');
487
+ logger?.log?.(`Recovery sync failed: ${(0, errors_1.getErrorMessage)(syncErr)}`, 'error');
389
488
  return null;
390
489
  }
391
490
  }
491
+ async function _recoverStartupSyncFailure({ chainOrders, manager, account, logger, triggerMessage, source }) {
492
+ return _recoverSyncFromChain({
493
+ chainOrders,
494
+ manager,
495
+ account,
496
+ logger,
497
+ source,
498
+ triggerMessage,
499
+ skipMessage: 'Startup: Recovery sync skipped - empty/truncated chain read is ambiguous (node may be lagging, ' +
500
+ 'or the get_full_accounts window omitted fresh orders); deferring to next reconcile cycle',
501
+ });
502
+ }
392
503
  function _refreshStartupUpdatePlans(updatePlans, chainOpenOrders) {
393
504
  if (!Array.isArray(updatePlans) || updatePlans.length === 0)
394
505
  return [];
@@ -440,7 +551,9 @@ function _prepareStartupUpdatePlan(plan, manager, logger) {
440
551
  async function _finalizeStartupUpdate({ manager, preparedUpdate }) {
441
552
  const { plan, parsedChain } = preparedUpdate;
442
553
  if (parsedChain && parsedChain.size > 0 && manager.accountant) {
443
- await manager.accountant.addToChainFree(plan.gridOrder.type, parsedChain.size, 'startup-align');
554
+ await manager._fundLock.acquire(async () => {
555
+ await manager.accountant.addToChainFree(plan.gridOrder.type, parsedChain.size, 'startup-align');
556
+ });
444
557
  }
445
558
  // Extract deferred_fee from the raw chain order object so the fee
446
559
  // lifecycle (cancel refunds, fill maker discounts) reconstructs
@@ -582,23 +695,126 @@ async function _executeStartupSequentialUpdateFallback({ updatePlans, chainOrder
582
695
  }
583
696
  return { executed, skipped, failed };
584
697
  }
585
- async function _createStartupOrderWithHandling({ chainOrders, account, privateKey, manager, gridOrder, orderLabel, dryRun, recovery, }) {
698
+ /**
699
+ * Verify whether an uncertain startup create actually landed on chain, and if
700
+ * so adopt it via a chain sync so the slot is registered with its real orderId.
701
+ * Returns the chain order id when the create landed.
702
+ * Returns the string 'unknown' when the chain state could NOT be verified
703
+ * (read failure, an empty read which is indistinguishable from a lagging
704
+ * node that missed the just-broadcast transaction, or a truncated read whose
705
+ * capped result set omits the fresh create) — the caller must NOT
706
+ * re-broadcast on 'unknown', or a landed order would be duplicated.
707
+ * Returns null only on AUTHORITATIVE absence: a successful non-empty,
708
+ * non-truncated read that contains no matching order. In that case
709
+ * re-broadcasting is safe.
710
+ * @private
711
+ */
712
+ async function _adoptPossiblyLandedCreate({ chainOrders, manager, account, gridOrder, }) {
586
713
  try {
587
- await _createOrderFromGrid({ chainOrders, account, privateKey, manager, gridOrder, dryRun });
714
+ // Truncated reads omit the newest limit orders (by_account index
715
+ // order) — exactly the create this check is verifying — and empty
716
+ // reads are indistinguishable from a lagging node that missed the
717
+ // just-broadcast transaction. Both are unverifiable; absence is only
718
+ // authoritative on a clean, non-empty read.
719
+ const freshChainOrders = await (0, chain_orders_1.readOpenOrdersGuarded)(chainOrders, (0, system_1.resolveAccountRef)(manager, account), {
720
+ deferEmpty: true,
721
+ timeoutMs: constants_1.TIMING.CONNECTION_TIMEOUT_MS,
722
+ });
723
+ if (freshChainOrders === null) {
724
+ return 'unknown';
725
+ }
726
+ const assets = manager?.assets;
727
+ if (!assets?.assetA || !assets?.assetB)
728
+ return 'unknown';
729
+ let matched = null;
730
+ for (const o of freshChainOrders) {
731
+ const parsed = (0, order_1.parseChainOrder)(o, assets);
732
+ if (!parsed || parsed.type !== gridOrder.type)
733
+ continue;
734
+ if (parsed.orderId && Array.from(manager.orders.values()).some((g) => g.orderId === parsed.orderId))
735
+ continue;
736
+ if (!(0, order_1.chainOrderMatchesSlot)(parsed, gridOrder, assets))
737
+ continue;
738
+ matched = o;
739
+ break;
740
+ }
741
+ if (!matched)
742
+ return null;
743
+ await manager.syncFromOpenOrders(freshChainOrders, {
744
+ // Accounting enabled: the landed order must be committed in the
745
+ // optimistic balances (VIRTUAL→ACTIVE locks its capital), or the
746
+ // funds appear free while locked on chain — same policy as
747
+ // _recoverStartupSyncFailure and the group-uncertain adoption.
748
+ skipAccounting: false,
749
+ source: 'startup-create-uncertain-adopt',
750
+ });
751
+ // Deduct the create fee: syncFromOpenOrders activates the slot and locks
752
+ // its capital but does not apply the on-chain create cost. The group
753
+ // batch path and the grid_reconcile adoption loop both apply it via
754
+ // _applySync(fee: createFee) — mirror that here so the optimistic
755
+ // balance reflects the real chain cost.
756
+ const btsFeeData = (0, math_1.getAssetFeesSafe)('BTS');
757
+ await manager._applySync({
758
+ gridOrderId: gridOrder.id,
759
+ chainOrderId: matched.id,
760
+ isPartialPlacement: false,
761
+ expectedType: gridOrder.type,
762
+ fee: btsFeeData?.createFee || 0,
763
+ }, 'createOrder');
764
+ return matched.id || null;
588
765
  }
589
- catch (err) {
590
- manager?.logger?.log?.(`Startup: Failed to create ${orderLabel}: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
591
- if (recovery && recovery.triggerMessage && recovery.source) {
592
- await _recoverStartupSyncFailure({
593
- chainOrders,
594
- manager,
595
- account,
596
- logger: manager?.logger,
597
- triggerMessage: recovery.triggerMessage,
598
- source: recovery.source,
599
- });
766
+ catch (adoptErr) {
767
+ manager?.logger?.log?.(`Startup: Uncertain-create adoption check failed for ${gridOrder?.id}: ${(0, errors_1.getErrorMessage)(adoptErr)}`, 'warn');
768
+ return 'unknown';
769
+ }
770
+ }
771
+ async function _createStartupOrderWithHandling({ chainOrders, account, privateKey, manager, gridOrder, orderLabel, dryRun, recovery, }) {
772
+ const maxAttempts = 2;
773
+ let failedWithUncertain = false;
774
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
775
+ try {
776
+ const chainOrderId = await _createOrderFromGrid({ chainOrders, account, privateKey, manager, gridOrder, dryRun });
777
+ if (chainOrderId)
778
+ return chainOrderId;
779
+ return null;
600
780
  }
781
+ catch (err) {
782
+ const isUncertain = err?.code === 'BROADCAST_UNCERTAIN' || err?.name === 'BroadcastUncertainError';
783
+ manager?.logger?.log?.(`Startup: Failed to create ${orderLabel} (attempt ${attempt}/${maxAttempts}${isUncertain ? ', uncertain broadcast' : ''}): ${(0, errors_1.getErrorMessage)(err)}`, 'error');
784
+ if (!isUncertain)
785
+ break;
786
+ failedWithUncertain = true;
787
+ const landed = await _adoptPossiblyLandedCreate({ chainOrders, manager, account, gridOrder });
788
+ if (landed && landed !== 'unknown') {
789
+ manager?.logger?.log?.(`Startup: Uncertain create for ${orderLabel} confirmed on chain (${landed}); adopted via chain sync`, 'warn');
790
+ return landed;
791
+ }
792
+ if (landed === 'unknown') {
793
+ // Chain state unverifiable (lagging/empty read). Re-broadcasting
794
+ // now could duplicate an order that actually landed; defer the
795
+ // create to the next startup reconcile cycle instead.
796
+ manager?.logger?.log?.(`Startup: Uncertain create for ${orderLabel} could not be verified on chain; deferring re-broadcast to next reconcile cycle (duplicate-order protection)`, 'warn');
797
+ break;
798
+ }
799
+ if (attempt < maxAttempts) {
800
+ manager?.logger?.log?.(`Startup: Uncertain create for ${orderLabel} not found on chain; retrying (authoritative absence verified)`, 'warn');
801
+ }
802
+ }
803
+ }
804
+ if (recovery && recovery.triggerMessage && recovery.source) {
805
+ await _recoverStartupSyncFailure({
806
+ chainOrders,
807
+ manager,
808
+ account,
809
+ logger: manager?.logger,
810
+ triggerMessage: recovery.triggerMessage,
811
+ source: recovery.source,
812
+ });
813
+ }
814
+ if (failedWithUncertain) {
815
+ manager?.logger?.log?.(`Startup: Create failed for ${orderLabel} after ${maxAttempts} attempt(s); slot kept for next startup reconcile cycle`, 'warn');
601
816
  }
817
+ return null;
602
818
  }
603
819
  // _extractBatchOperationResults — thin wrapper over shared utility,
604
820
  // returns empty array (not null) for callers that iterate directly.
@@ -615,7 +831,7 @@ function _resolveGroupRecovery(group, fallbackMessage, fallbackSource) {
615
831
  }
616
832
  async function _executeStartupCreateGroupBatch({ group, chainOrders, account, privateKey, manager, dryRun, groupIndex, totalGroups, }) {
617
833
  if (!Array.isArray(group) || group.length === 0 || dryRun)
618
- return;
834
+ return [];
619
835
  if (typeof chainOrders?.buildCreateOrderOp !== 'function' || typeof chainOrders?.executeBatch !== 'function') {
620
836
  throw new Error('chainOrders does not support batch create operations');
621
837
  }
@@ -651,13 +867,18 @@ async function _executeStartupCreateGroupBatch({ group, chainOrders, account, pr
651
867
  prepared.push({ plan, op: buildResult.op });
652
868
  }
653
869
  if (prepared.length === 0)
654
- return;
870
+ return [];
655
871
  const recovery = _resolveGroupRecovery(group, `Startup: Triggering recovery sync after create group ${groupIndex + 1}/${totalGroups} failure`, 'startupCreateGroupFailure');
872
+ const createdOrderIds = [];
656
873
  try {
657
874
  logger?.log?.(`Startup: Broadcasting create group ${groupIndex + 1}/${totalGroups} in single batch (${prepared.length} op${prepared.length > 1 ? 's' : ''})`, 'info');
658
875
  const batchResult = await chainOrders.executeBatch(account, privateKey, prepared.map((p) => p.op));
659
876
  const opResults = _extractBatchOperationResults(batchResult);
660
877
  const btsFeeData = (0, math_1.getAssetFees)('BTS');
878
+ // Phase A: Extract ALL chain order IDs from batch results first, capturing
879
+ // every on-chain ID before any _applySync call. This ensures createdOrderIds
880
+ // is complete even if a subsequent _applySync throws mid-loop.
881
+ const batchResults = [];
661
882
  let missingChainOrderId = false;
662
883
  for (let i = 0; i < prepared.length; i++) {
663
884
  const chainOrderId = opResults[i] && opResults[i][1];
@@ -665,8 +886,17 @@ async function _executeStartupCreateGroupBatch({ group, chainOrders, account, pr
665
886
  if (!chainOrderId) {
666
887
  logger?.log?.(`Startup: create result missing chainOrderId for ${plan.orderLabel}`, 'error');
667
888
  missingChainOrderId = true;
668
- continue;
669
889
  }
890
+ else {
891
+ createdOrderIds.push(chainOrderId);
892
+ }
893
+ batchResults.push({ chainOrderId, plan });
894
+ }
895
+ // Phase B: Register each created order with the manager. All IDs are
896
+ // already in createdOrderIds, so _applySync failures won't lose them.
897
+ for (const { chainOrderId, plan } of batchResults) {
898
+ if (!chainOrderId)
899
+ continue;
670
900
  await manager._applySync({
671
901
  gridOrderId: plan.gridOrder.id,
672
902
  chainOrderId,
@@ -687,16 +917,104 @@ async function _executeStartupCreateGroupBatch({ group, chainOrders, account, pr
687
917
  }
688
918
  }
689
919
  catch (err) {
690
- logger?.log?.(`Startup: Failed to create group ${groupIndex + 1}/${totalGroups}: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
691
- await _recoverStartupSyncFailure({
692
- chainOrders,
693
- manager,
694
- account,
695
- logger,
696
- triggerMessage: recovery.triggerMessage,
697
- source: recovery.source,
698
- });
920
+ const isUncertain = err?.code === 'BROADCAST_UNCERTAIN' || err?.name === 'BroadcastUncertainError';
921
+ logger?.log?.(`Startup: Failed to create group ${groupIndex + 1}/${totalGroups}${isUncertain ? ' (uncertain broadcast)' : ''}: ${(0, errors_1.getErrorMessage)(err)}`, 'error');
922
+ if (isUncertain) {
923
+ // The batch may have landed despite the deadline. Verify on chain and
924
+ // adopt any landed orders via chain sync instead of blindly
925
+ // re-broadcasting (duplicate order risk).
926
+ logger?.log?.(`Startup: Verifying uncertain create group ${groupIndex + 1}/${totalGroups} on chain; adopting any landed orders`, 'warn');
927
+ try {
928
+ // An empty/truncated verification read is ambiguous, NOT
929
+ // proof that nothing landed: a truncated get_full_accounts
930
+ // window omits the freshest creates (exactly this batch's),
931
+ // and an empty snapshot may be a node lagging behind the
932
+ // just-broadcast transaction. Treating it as "nothing
933
+ // landed" would let a later pass re-create (duplicate) or
934
+ // cancel as surplus the very orders this batch placed.
935
+ // Mirror the single-create sibling: defer through the
936
+ // guarded recovery sync, which re-reads and only syncs on
937
+ // an authoritative snapshot (deferring to the next
938
+ // reconcile cycle otherwise).
939
+ const freshChainOrders = await (0, chain_orders_1.readOpenOrdersGuarded)(chainOrders, (0, system_1.resolveAccountRef)(manager, account), {
940
+ log: (message, level) => logger?.log?.(message, level),
941
+ label: 'STARTUP',
942
+ deferEmpty: true,
943
+ timeoutMs: constants_1.TIMING.CONNECTION_TIMEOUT_MS,
944
+ skipMessage: (kind) => `Startup: Uncertain create group ${groupIndex + 1}/${totalGroups} verification read is ` +
945
+ `${kind}; cannot confirm whether the batch landed - ` +
946
+ `deferring to guarded recovery sync (duplicate-order protection)`,
947
+ });
948
+ if (freshChainOrders === null) {
949
+ await _recoverStartupSyncFailure({
950
+ chainOrders,
951
+ manager,
952
+ account,
953
+ logger,
954
+ triggerMessage: recovery.triggerMessage,
955
+ source: recovery.source,
956
+ });
957
+ }
958
+ else {
959
+ await manager.syncFromOpenOrders(freshChainOrders, {
960
+ // Accounting enabled: the batch's created orders were
961
+ // never registered (no _applySync ran), so the adoption
962
+ // must lock their capital. skipAccounting:true would
963
+ // leave the optimistic balances drifted.
964
+ skipAccounting: false,
965
+ source: 'startup-create-group-uncertain-adopt',
966
+ });
967
+ // Complete the create accounting (fees + order registration)
968
+ // for every group plan whose slot was adopted, and register
969
+ // the adopted IDs so the final refresh never cancels them
970
+ // as surplus.
971
+ const btsFeeData = (0, math_1.getAssetFeesSafe)('BTS');
972
+ for (const item of prepared) {
973
+ const plan = item?.plan;
974
+ const slot = plan?.gridOrder?.id ? manager.orders.get(plan.gridOrder.id) : null;
975
+ if (!slot?.orderId)
976
+ continue;
977
+ try {
978
+ await manager._applySync({
979
+ gridOrderId: plan.gridOrder.id,
980
+ chainOrderId: slot.orderId,
981
+ isPartialPlacement: false,
982
+ expectedType: plan.gridOrder.type,
983
+ fee: btsFeeData?.createFee || 0,
984
+ }, 'createOrder');
985
+ createdOrderIds.push(slot.orderId);
986
+ logger?.log?.(`Startup: Uncertain create for ${plan.orderLabel} confirmed on chain (${slot.orderId}); adopted + accounted`, 'warn');
987
+ }
988
+ catch (applyErr) {
989
+ logger?.log?.(`Startup: Adoption accounting failed for ${plan.orderLabel}: ${(0, errors_1.getErrorMessage)(applyErr)}`, 'warn');
990
+ }
991
+ }
992
+ }
993
+ }
994
+ catch (verifyErr) {
995
+ logger?.log?.(`Startup: Uncertain group verification failed: ${(0, errors_1.getErrorMessage)(verifyErr)}; falling back to recovery sync`, 'error');
996
+ await _recoverStartupSyncFailure({
997
+ chainOrders,
998
+ manager,
999
+ account,
1000
+ logger,
1001
+ triggerMessage: recovery.triggerMessage,
1002
+ source: recovery.source,
1003
+ });
1004
+ }
1005
+ }
1006
+ else {
1007
+ await _recoverStartupSyncFailure({
1008
+ chainOrders,
1009
+ manager,
1010
+ account,
1011
+ logger,
1012
+ triggerMessage: recovery.triggerMessage,
1013
+ source: recovery.source,
1014
+ });
1015
+ }
699
1016
  }
1017
+ return createdOrderIds;
700
1018
  }
701
1019
  function _buildOutsideInCreateGroups(createPlans) {
702
1020
  return (0, order_1.buildOutsideInPairGroups)(createPlans, {
@@ -708,8 +1026,9 @@ function _buildOutsideInCreateGroups(createPlans) {
708
1026
  async function _executePlannedStartupCreates({ createPlans, chainOrders, account, privateKey, manager, dryRun, }) {
709
1027
  const logger = manager?.logger;
710
1028
  const groups = _buildOutsideInCreateGroups(createPlans);
1029
+ const createdOrderIds = new Set();
711
1030
  if (groups.length === 0)
712
- return;
1031
+ return createdOrderIds;
713
1032
  logger?.log?.(`Startup: Executing ${createPlans.length} planned create(s) in ${groups.length} outside->center group(s)`, 'info');
714
1033
  for (let i = 0; i < groups.length; i++) {
715
1034
  const group = groups[i];
@@ -717,7 +1036,7 @@ async function _executePlannedStartupCreates({ createPlans, chainOrders, account
717
1036
  logger?.log?.(`Startup: Create group ${i + 1}/${groups.length} (${labels})`, 'info');
718
1037
  const canBatchCreate = typeof chainOrders?.buildCreateOrderOp === 'function' && typeof chainOrders?.executeBatch === 'function';
719
1038
  if (group.length > 1 && canBatchCreate) {
720
- await _executeStartupCreateGroupBatch({
1039
+ const batchIds = await _executeStartupCreateGroupBatch({
721
1040
  group,
722
1041
  chainOrders,
723
1042
  account,
@@ -727,13 +1046,15 @@ async function _executePlannedStartupCreates({ createPlans, chainOrders, account
727
1046
  groupIndex: i,
728
1047
  totalGroups: groups.length,
729
1048
  });
1049
+ for (const id of batchIds)
1050
+ createdOrderIds.add(id);
730
1051
  continue;
731
1052
  }
732
1053
  if (group.length > 1 && !canBatchCreate) {
733
1054
  logger?.log?.('Startup: Batch create helpers unavailable; falling back to sequential creates for this group', 'warn');
734
1055
  }
735
1056
  for (const plan of group) {
736
- await _createStartupOrderWithHandling({
1057
+ const chainOrderId = await _createStartupOrderWithHandling({
737
1058
  chainOrders,
738
1059
  account,
739
1060
  privateKey,
@@ -743,8 +1064,14 @@ async function _executePlannedStartupCreates({ createPlans, chainOrders, account
743
1064
  dryRun,
744
1065
  recovery: plan.recovery,
745
1066
  });
1067
+ if (chainOrderId)
1068
+ createdOrderIds.add(chainOrderId);
746
1069
  }
747
1070
  }
1071
+ const failedCount = Math.max(0, createPlans.length - createdOrderIds.size);
1072
+ logger?.log?.(`Startup: Create execution summary: ${createdOrderIds.size}/${createPlans.length} planned create(s) placed ` +
1073
+ `across ${groups.length} group(s)${failedCount > 0 ? `, ${failedCount} failed/skipped (see error logs)` : ''}`, failedCount > 0 ? 'warn' : 'info');
1074
+ return createdOrderIds;
748
1075
  }
749
1076
  async function _reconcileStartupSide({ orderType, targetCount, chainSideOrders, unmatchedSideOrders, manager, chainOrders, account, privateKey, dryRun, plannedCreates, plannedUpdates, plannedCancels, planOnly = false, }) {
750
1077
  const logger = manager?.logger;