dexbot 1.5.1 → 1.5.3

package/dist/modules/order/sync_engine.js

@@ -95,7 +95,7 @@
  */
 import { ORDER_TYPES, ORDER_STATES, TIMING, BTS_PRECISION } from '../constants.js';
 import * as Format from './format.js';
-import { lookupAsset } from './utils/system.js';
+import { lookupAsset, sleep, resolveAccountRef } from './utils/system.js';
 import * as chainOrders from '../chain_orders.js';
 import * as client from '../bitshares_client.js';
 const { BitShares } = client;
@@ -106,6 +106,73 @@ import { parseChainOrder, findMatchingGridOrderByOpenOrder, applyChainSizeToGrid
 import { parseSlotIndex } from './utils/slot.js';
 import { resolveProcessedFillPersistenceMode } from './processed_fill_store.js';
 import { getErrorMessage } from '../utils/errors.js';
+/**
+ * Confirming re-read for the suspect-empty-read guard: after
+ * SYNC_SUSPECT_EMPTY_READ_LIMIT consecutive 0-order snapshots with placed
+ * grid orders still present, one more read after
+ * SYNC_EMPTY_READ_CONFIRM_DELAY_MS decides whether the account is genuinely
+ * empty. A lagging node serves consecutive empties, so the count alone is
+ * not authoritative.
+ *
+ * Test seam: when manager._confirmEmptyReadFn is a function it is used
+ * instead of the chain read (must resolve to a raw order array, or null
+ * when the re-read itself is ambiguous). A throwing seam is 'ambiguous'.
+ *
+ * @param {Object} mgr - OrderManager instance (accountId, config, logger).
+ * @returns {Promise<string>} 'confirmed' (still empty — accept),
+ *   'contradicted' (non-empty — reset counter, refuse this round),
+ *   'ambiguous' (truncated/read-error — refuse, re-confirm next round), or
+ *   'unavailable' (no chain identity or dry-run — caller falls back to the
+ *   legacy count-based acceptance, preserving unit-test semantics).
+ */
+async function confirmSuspectEmptyRead(mgr) {
+    try {
+        if (mgr?.config?.dryRun)
+            return 'unavailable';
+        const seam = mgr?._confirmEmptyReadFn;
+        const delay = Math.max(0, Number(TIMING.SYNC_EMPTY_READ_CONFIRM_DELAY_MS) || 0);
+        if (typeof seam === 'function') {
+            if (delay > 0)
+                await sleep(delay);
+            let fresh = null;
+            try {
+                fresh = await seam();
+            }
+            catch {
+                return 'ambiguous';
+            }
+            if (fresh === null || fresh === undefined)
+                return 'ambiguous';
+            if (Array.isArray(fresh) && fresh.length > 0)
+                return 'contradicted';
+            return 'confirmed';
+        }
+        let accountRef = null;
+        try {
+            accountRef = resolveAccountRef(mgr, null);
+        }
+        catch {
+            return 'unavailable';
+        }
+        if (!accountRef)
+            return 'unavailable';
+        if (delay > 0)
+            await sleep(delay);
+        const fresh = await chainOrders.readOpenOrdersGuarded(chainOrders, accountRef, {
+            log: (message, level) => mgr?.logger?.log?.(message, level),
+            label: 'SYNC-CONFIRM',
+            detail: 'suspect-empty confirm re-read',
+        });
+        if (fresh === null)
+            return 'ambiguous';
+        if (Array.isArray(fresh) && fresh.length > 0)
+            return 'contradicted';
+        return 'confirmed';
+    }
+    catch {
+        return 'ambiguous';
+    }
+}
 function describeNearestAdoptionCandidates(mgr, chainOrder, precision, calcTolerance, matchedGridOrderIds = null) {
     if (!mgr?.orders || !chainOrder || typeof precision !== 'number')
         return 'candidate diagnostics unavailable';
@@ -244,8 +311,15 @@ async function adoptChainOrderIntoSlot(mgr, slot, chainOrder, chainOrderId, rawC
         else {
             const spreadOrder = convertToSpreadPlaceholder(bestMatch);
             const applied = await mgr._applyOrderUpdate(spreadOrder, 'sync-pass2-filled', { skipAccounting: skipAccounting, fee: 0 });
-            if (applied === false)
+            if (applied === false) {
+                // Rejected adoption must not poison the slot for the next
+                // chain order: the slot was marked matched above, so release
+                // it — B's adoption re-fails the same validation and gets
+                // the existing cancelOnly handling if the rejection was
+                // slot-specific.
+                matchedGridOrderIds.delete(bestMatch.id);
                 return false;
+            }
             filledOrders.push({ ...bestMatch });
             updatedOrders.push(spreadOrder);
             chainOrderIdsOnGrid.add(chainOrderId);
@@ -256,8 +330,12 @@ async function adoptChainOrderIntoSlot(mgr, slot, chainOrder, chainOrderId, rawC
         bestMatch.state = ORDER_STATES.PARTIAL;
     }
     const applied = await mgr._applyOrderUpdate(bestMatch, 'sync-pass2-orphan', { skipAccounting: skipAccounting, fee: 0 });
-    if (applied === false)
+    if (applied === false) {
+        // Same un-poisoning as the filled path above: the slot counts as
+        // matched only once the update lands.
+        matchedGridOrderIds.delete(bestMatch.id);
         return false;
+    }
     updatedOrders.push(bestMatch);
     chainOrderIdsOnGrid.add(chainOrderId);
     return true;
@@ -581,7 +659,8 @@ class SyncEngine {
         // ("phantom" resets). Refuse to reconcile on a
         // suspect empty; the guard is self-expiring: after
         // TIMING.SYNC_SUSPECT_EMPTY_READ_LIMIT consecutive empty reads the
-        // account really is empty and the sync accepts it. Any non-empty read
+        // account really is empty and the sync accepts it — but only after one
+        // confirming re-read (see below). Any non-empty read
         // resets the counter.
         if (parsedChainOrders.size === 0) {
             const gridOrderIds = Array.from(mgr.orders.values()).filter((o) => o?.orderId).length;
@@ -597,7 +676,33 @@ class SyncEngine {
                         `refusing reconciliation (phantom protection); accepting after ${limit} consecutive empties or next non-empty read`, 'warn');
                     return { filledOrders: [], updatedOrders: [], ordersNeedingCorrection: [], unmatchedChainOrders: [] };
                 }
-                mgr.logger?.log?.(`[SYNC] Empty read confirmed after ${suspect.count} consecutive attempts — reconciling to empty account`, 'warn');
+                // Limit reached: a lagging node serves CONSECUTIVE 0-order
+                // snapshots, so the count alone is not authoritative. Require
+                // one confirming re-read after SYNC_EMPTY_READ_CONFIRM_DELAY_MS
+                // before reconciling to empty; a contradicted re-read
+                // (non-empty) resets the counter and refuses this round (the
+                // next sync reconciles against the fresh snapshot), while an
+                // ambiguous re-read (truncated/read-error) refuses and retries
+                // the confirm on the next empty round.
+                const confirm = await confirmSuspectEmptyRead(mgr);
+                if (confirm === 'contradicted') {
+                    mgr.logger?.log?.(`[SYNC] Suspect empty read contradicted by confirm re-read with ${gridOrderIds} grid orderIds — ` +
+                        `resetting empty-read counter; reconciling on the next non-empty sync`, 'warn');
+                    mgr._suspectEmptyReads = { count: 0, firstAt: 0 };
+                    return { filledOrders: [], updatedOrders: [], ordersNeedingCorrection: [], unmatchedChainOrders: [] };
+                }
+                if (confirm === 'ambiguous') {
+                    mgr.logger?.log?.(`[SYNC] Suspect empty read confirm re-read ambiguous with ${gridOrderIds} grid orderIds — ` +
+                        `refusing reconciliation; re-confirming on the next empty read`, 'warn');
+                    mgr._suspectEmptyReads = { count: limit, firstAt: suspect.firstAt || Date.now() };
+                    return { filledOrders: [], updatedOrders: [], ordersNeedingCorrection: [], unmatchedChainOrders: [] };
+                }
+                // 'confirmed' (still empty on re-read) or 'unavailable' (no
+                // chain identity / dry-run — legacy count-based acceptance,
+                // preserving unit-test semantics): reconcile to empty.
+                mgr.logger?.log?.(`[SYNC] Empty read confirmed after ${suspect.count} consecutive attempts` +
+                    (confirm === 'unavailable' ? ' (no chain identity for confirm re-read)' : ' (+ confirming re-read)') +
+                    ` — reconciling to empty account`, 'warn');
                 mgr._suspectEmptyReads = { count: 0, firstAt: 0 };
             }
             else {
@@ -843,9 +948,17 @@ class SyncEngine {
                             && toFiniteNumber(reboundOrder.rawOnChain?.for_sale, 0) > 0) {
                             reboundOrder.state = ORDER_STATES.PARTIAL;
                         }
+                        const applied = await mgr._applyOrderUpdate(reboundOrder, 'sync-pass1-duplicate-swap', { skipAccounting: skipAccounting, fee: 0 });
+                        if (applied === false) {
+                            // Rejected rebind: the slot keeps its old binding
+                            // (set ops must run only after the apply lands), so
+                            // the old orderId stays matched on-grid and the swap
+                            // candidate stays unmatched → pass 2 cancels it.
+                            mgr.logger?.log?.(`[SYNC] Size tiebreak for ${gridOrder.id}: rebind to ${swapMatch.id} rejected — slot keeps ${gridOrder.orderId}; duplicate falls to pass 2`, 'warn');
+                            continue;
+                        }
                         chainOrderIdsOnGrid.delete(gridOrder.orderId);
                         chainOrderIdsOnGrid.add(swapMatch.id);
-                        await mgr._applyOrderUpdate(reboundOrder, 'sync-pass1-duplicate-swap', { skipAccounting: skipAccounting, fee: 0 });
                         updatedOrders.push(reboundOrder);
                         mgr.logger?.log?.(`[SYNC] Size tiebreak for ${gridOrder.id}: tracked order ${gridOrder.orderId} ` +
                             `(chain size ${chainOrder.size}) disagrees with booked size ${gridOrder.size}, but ` +
@@ -869,11 +982,23 @@ class SyncEngine {
                         updatedOrder.state = (chainSizeInt < currentSizeInt)
                             ? ORDER_STATES.PARTIAL
                             : gridOrder.state;
-                        await mgr._applyOrderUpdate(updatedOrder, 'sync-pass1-partial', { skipAccounting: skipAccounting, fee: 0 });
+                        const partialApplied = await mgr._applyOrderUpdate(updatedOrder, 'sync-pass1-partial', { skipAccounting: skipAccounting, fee: 0 });
+                        if (partialApplied === false) {
+                            // Rejected resize: the slot keeps its booked size;
+                            // divergence re-detects on the next sync.
+                            mgr.logger?.log?.(`[SYNC] Partial-size update for ${gridOrder.id} rejected — slot keeps booked size ${gridOrder.size}`, 'warn');
+                        }
                     }
                     else {
                         const spreadOrder = convertToSpreadPlaceholder(gridOrder);
-                        await mgr._applyOrderUpdate(spreadOrder, 'sync-pass1-filled', { skipAccounting: skipAccounting, fee: 0 });
+                        const filledApplied = await mgr._applyOrderUpdate(spreadOrder, 'sync-pass1-filled', { skipAccounting: skipAccounting, fee: 0 });
+                        if (filledApplied === false) {
+                            // Rejected virtualization must not book a fill:
+                            // neither the filled order nor the placeholder
+                            // update leaves this path.
+                            mgr.logger?.log?.(`[SYNC] Filled virtualization for ${gridOrder.id} rejected — fill NOT booked`, 'error');
+                            continue;
+                        }
                         // Push the filled order with its REAL side (chain order
                         // type), not the SPREAD placeholder: downstream fill
                         // processing (deriveTargetBoundary) derives boundary
@@ -927,7 +1052,14 @@ class SyncEngine {
                 const currentGridOrder = mgr.orders.get(gridOrder.id) || gridOrder;
                 const hadOrderId = Boolean(currentGridOrder?.orderId);
                 const spreadOrder = convertToSpreadPlaceholder(currentGridOrder);
-                await mgr._applyOrderUpdate(spreadOrder, 'sync-cleanup-phantom', { skipAccounting: skipAccounting, fee: 0 });
+                const phantomApplied = await mgr._applyOrderUpdate(spreadOrder, 'sync-cleanup-phantom', { skipAccounting: skipAccounting, fee: 0 });
+                if (phantomApplied === false) {
+                    // Rejected virtualization books no phantom fill: the slot
+                    // keeps its order and the disappearance re-detects next
+                    // sync.
+                    mgr.logger?.log?.(`[SYNC] Phantom cleanup for ${gridOrder.id} rejected — fill NOT booked`, 'warn');
+                    continue;
+                }
                 // Only genuine disappearances (had orderId) count as fills.
                 if (hadOrderId) {
                     // A SPREAD slot can carry an on-chain order (spread-correction
@@ -975,6 +1107,19 @@ class SyncEngine {
                 const slotId = `slot-${idx}`;
                 const gapSlots = genesis.gapSlots ?? mgr._gapSlots ?? 0;
                 const boundaryIdx = mgr.boundaryIdx;
+                // Pre-boundary sync: gap geometry is unknown, so adoption is
+                // deferred entirely — touch nothing (no adopt, no cancelOnly).
+                // The orphan stays visible to the crossing guards and the
+                // validate orphan layer via _lastUnmatchedChainOrders and is
+                // re-evaluated once the boundary commits. Accepted cost: a
+                // legitimate in-rail orphan waits one sync cycle
+                // post-boundary-commit before adoption. Strictly better than
+                // adopting a gap stray into the wrong slot.
+                if (boundaryIdx == null || !Number.isFinite(Number(boundaryIdx))) {
+                    unmatchedChainOrders.push({ chainOrderId, type: chainOrder.type, price: chainOrder.price, size: chainOrder.size, raw: rawChainOrders.get(chainOrderId), reason: 'boundary-unknown-deferred', candidateSlotId: slotId });
+                    mgr.logger?.log?.(`[SYNC] Orphaned chain order ${chainOrderId} (${chainOrder.type}, price=${chainOrder.price}, size=${chainOrder.size}) — NOT adopted: boundary unknown, deferred until boundary commits (nearest slot ${slotId})`, 'warn');
+                    continue;
+                }
                 // Duplicate-price guard becomes slotId equality: if placed order already occupies this slot
                 const occupying = mgr.orders.get(slotId);
                 if (occupying && isOrderPlaced(occupying) && occupying.type === chainOrder.type) {
@@ -984,8 +1129,9 @@ class SyncEngine {
                     queueCorrection({ gridOrder: occupying, chainOrderId, expectedPrice: chainOrder.price, size: chainOrder.size, type: chainOrder.type, isSurplus: true, cancelOnly: true });
                     continue;
                 }
-                // Gap exclusion: nearest slot in SPREAD gap → no adopt. When boundaryIdx == null (pre-boundary sync) the gap is unknown so we allow adoption — the next reconcile will relocate gap stray per §10 table's cancelOnly intent (documented hole).
-                if (boundaryIdx != null && Number.isFinite(Number(boundaryIdx))) {
+                // Gap exclusion: nearest slot in SPREAD gap → no adopt (boundary
+                // is known here — the pre-boundary case continued above).
+                {
                     const inRail = isSlotInRail(boundaryIdx, gapSlots, chainOrder.type, { id: slotId });
                     if (!inRail) {
                         unmatchedChainOrders.push({ chainOrderId, type: chainOrder.type, price: chainOrder.price, size: chainOrder.size, raw: rawChainOrders.get(chainOrderId), reason: 'no-available-nearest-slot', candidateSlotId: slotId });
@@ -1799,6 +1945,11 @@ class SyncEngine {
         switch (source) {
             case 'createOrder': {
                 const { gridOrderId, chainOrderId, isPartialPlacement, expectedType, fee } = chainData;
+                // Optional placement descriptor for unknown-id recovery. Callers
+                // pass the placed slot's price/size/type so a grid reset racing
+                // the broadcast can't strand the live chain order untracked.
+                // Callers that omit it take the no-descriptor error path.
+                const placedDescriptor = chainData.order ?? null;
                 const runCreate = async () => {
                     // Lock order to prevent concurrent modifications during state transition
                     mgr.lockOrders([gridOrderId]);
@@ -1861,6 +2012,68 @@ class SyncEngine {
                                 fee: actualFee
                             });
                         }
+                        else {
+                            // Unknown grid id: the broadcast landed but master no
+                            // longer holds the slot (grid reset/regen raced the
+                            // CREATE, or a first placement for an id never
+                            // projected via projectTargetToWorkingGrid). The old
+                            // code dropped the linkage silently, so the next
+                            // cycle re-placed the same level (duplicate CREATEs,
+                            // funds locked xN). Materialize-or-error: rebind when
+                            // the chain id is already tracked, materialize the
+                            // slot when the caller supplied its descriptor, and
+                            // log an error otherwise so the order never vanishes
+                            // quietly again.
+                            const rebound = Array.from(mgr.orders.values()).find((o) => o.orderId === chainOrderId);
+                            if (rebound) {
+                                mgr.logger?.log?.(`[SYNC] createOrder for unknown grid order ${gridOrderId}: chain order ${chainOrderId} already tracked on ${rebound.id} — linkage up to date, skipping`, 'warn');
+                            }
+                            else {
+                                const rawType = placedDescriptor?.type ?? expectedType;
+                                let materializeType = (rawType === ORDER_TYPES.BUY || rawType === ORDER_TYPES.SELL)
+                                    ? rawType
+                                    : null;
+                                const descriptorPrice = toFiniteNumber(placedDescriptor?.price, NaN);
+                                if (!materializeType && Number.isFinite(descriptorPrice)) {
+                                    materializeType = resolveSpreadOrderSide(descriptorPrice, mgr.config.startPrice);
+                                }
+                                const descriptorSize = toFiniteNumber(placedDescriptor?.size, NaN);
+                                const hasDescriptor = !!materializeType
+                                    && Number.isFinite(descriptorPrice) && descriptorPrice > 0
+                                    && Number.isFinite(descriptorSize) && descriptorSize > 0;
+                                if (!hasDescriptor) {
+                                    mgr.logger?.log?.(`[SYNC] createOrder linkage LOST: grid order ${gridOrderId} not in master — chain order ${chainOrderId} placed but untracked (no placement descriptor; next readOpenOrders sync must adopt it as an orphan before the slot re-places)`, 'error');
+                                }
+                                else {
+                                    // Minimal slot shape: the raw chain object
+                                    // isn't available here, so rawOnChain and
+                                    // slot-scheme metadata are absent. The next
+                                    // readOpenOrders pass repopulates rawOnChain
+                                    // from the live order (slot-scheme ids only).
+                                    const materializedOrder = {
+                                        id: gridOrderId,
+                                        type: materializeType,
+                                        price: descriptorPrice,
+                                        size: descriptorSize,
+                                        state: isPartialPlacement ? ORDER_STATES.PARTIAL : ORDER_STATES.ACTIVE,
+                                        orderId: chainOrderId,
+                                    };
+                                    if (chainData.deferredFee !== undefined && chainData.deferredFee !== null) {
+                                        materializedOrder.btsFeeState = { deferredFee: Math.max(0, chainData.deferredFee) };
+                                    }
+                                    const materialized = await mgr._applyOrderUpdate(materializedOrder, 'createOrder-unknown-id-adopt', {
+                                        skipAccounting: chainData.skipAccounting || false,
+                                        fee: fee,
+                                    });
+                                    if (materialized === false) {
+                                        mgr.logger?.log?.(`[SYNC] createOrder linkage FAILED: grid order ${gridOrderId} not in master and materializing chain order ${chainOrderId} (${materializeType} @${descriptorPrice} x${descriptorSize}) was rejected — chain order left untracked`, 'error');
+                                    }
+                                    else {
+                                        mgr.logger?.log?.(`[SYNC] createOrder for unknown grid order ${gridOrderId}: materialized ${materializeType} @${descriptorPrice} x${descriptorSize} -> ${chainOrderId} (master lost the slot mid-broadcast)`, 'warn');
+                                    }
+                                }
+                            }
+                        }
                     }
                     finally {
                         mgr.unlockOrders([gridOrderId]);
@@ -1912,6 +2125,13 @@ class SyncEngine {
                                     fee: btsFeeData?.cancelFee || 0
                                 });
                             }
+                            else {
+                                // Slot vanished or was rebound between lookup and
+                                // lock — never skip quietly: the next
+                                // readOpenOrders sync reconciles the slot, but the
+                                // race must be visible when it happens.
+                                mgr.logger?.log?.(`[SYNC] cancelOrder for ${orderId}: slot ${gridOrder.id} changed underneath (missing or rebound to ${currentGridOrder?.orderId ?? 'none'}) — linkage skipped, next sync reconciles`, 'warn');
+                            }
                         }
                         finally {
                             mgr.unlockOrders(orderIds);