dexbot 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +3 -3
- package/analysis/README.md +33 -13
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/account_resolver.js +5 -6
- package/dist/analysis/analyze_liquidity.js +15 -4
- package/dist/analysis/bot_key_utils.d.ts +44 -1
- package/dist/analysis/bot_key_utils.js +76 -7
- package/dist/analysis/fills_cache.js +2 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -19
- package/dist/analysis/grid_correction_check.js +12 -30
- package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
- package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
- package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
- package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
- package/dist/analysis/liquidity/liquidity_view.js +11 -0
- package/dist/analysis/resolve_bot_accounts.js +7 -6
- package/dist/analysis/trade_profitability.js +2 -1
- package/dist/credential-daemon.js +13 -2
- package/dist/dexbot.js +4 -4
- package/dist/modules/account_bots.d.ts +12 -1
- package/dist/modules/account_bots.js +7 -5
- package/dist/modules/account_orders.d.ts +12 -0
- package/dist/modules/account_orders.js +30 -0
- package/dist/modules/bitshares-native/transport.js +24 -6
- package/dist/modules/bitshares_client.d.ts +24 -6
- package/dist/modules/bitshares_client.js +44 -7
- package/dist/modules/bot_settings.d.ts +2 -0
- package/dist/modules/constants.d.ts +28 -3
- package/dist/modules/constants.js +79 -10
- package/dist/modules/dexbot_class.d.ts +23 -6
- package/dist/modules/dexbot_class.js +44 -0
- package/dist/modules/dexbot_cow_runtime.d.ts +128 -8
- package/dist/modules/dexbot_cow_runtime.js +688 -188
- package/dist/modules/dexbot_fill_runtime.js +124 -35
- package/dist/modules/dexbot_maintenance_runtime.d.ts +1 -1
- package/dist/modules/dexbot_maintenance_runtime.js +259 -79
- package/dist/modules/dexbot_startup_runtime.js +32 -20
- package/dist/modules/dexbot_state_recovery.d.ts +33 -1
- package/dist/modules/dexbot_state_recovery.js +167 -5
- package/dist/modules/node_connect_policy.d.ts +8 -1
- package/dist/modules/node_connect_policy.js +18 -2
- package/dist/modules/node_flap_score.d.ts +87 -0
- package/dist/modules/node_flap_score.js +124 -0
- package/dist/modules/node_health_cache.d.ts +40 -3
- package/dist/modules/node_health_cache.js +66 -20
- package/dist/modules/node_manager.d.ts +101 -19
- package/dist/modules/node_manager.js +255 -65
- package/dist/modules/order/accounting.d.ts +9 -4
- package/dist/modules/order/accounting.js +131 -19
- package/dist/modules/order/async_lock.d.ts +7 -1
- package/dist/modules/order/async_lock.js +7 -1
- package/dist/modules/order/export.js +25 -18
- package/dist/modules/order/fill_ledger.d.ts +99 -0
- package/dist/modules/order/fill_ledger.js +103 -0
- package/dist/modules/order/format.d.ts +24 -10
- package/dist/modules/order/format.js +30 -10
- package/dist/modules/order/grid.d.ts +75 -18
- package/dist/modules/order/grid.js +289 -72
- package/dist/modules/order/grid_reconcile.d.ts +5 -1
- package/dist/modules/order/grid_reconcile.js +333 -210
- package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
- package/dist/modules/order/grid_reconcile_internal.js +223 -91
- package/dist/modules/order/index.d.ts +18 -5
- package/dist/modules/order/index.js +5 -5
- package/dist/modules/order/lifecycle.d.ts +353 -0
- package/dist/modules/order/lifecycle.js +1109 -0
- package/dist/modules/order/manager.d.ts +277 -20
- package/dist/modules/order/manager.js +680 -64
- package/dist/modules/order/placement_lock.d.ts +102 -0
- package/dist/modules/order/placement_lock.js +222 -0
- package/dist/modules/order/strategy.js +13 -1
- package/dist/modules/order/sync_engine.d.ts +74 -114
- package/dist/modules/order/sync_engine.js +254 -198
- package/dist/modules/order/utils/math.d.ts +29 -4
- package/dist/modules/order/utils/math.js +61 -4
- package/dist/modules/order/utils/order.d.ts +106 -37
- package/dist/modules/order/utils/order.js +256 -81
- package/dist/modules/order/utils/system.d.ts +99 -6
- package/dist/modules/order/utils/system.js +249 -35
- package/dist/modules/order/utils/validate.d.ts +7 -1
- package/dist/modules/order/utils/validate.js +48 -2
- package/dist/modules/outcome.d.ts +54 -0
- package/dist/modules/outcome.js +51 -0
- package/dist/modules/settings_merge.d.ts +4 -0
- package/dist/modules/settings_merge.js +9 -0
- package/dist/modules/timing_policy.d.ts +82 -0
- package/dist/modules/timing_policy.js +244 -0
- package/dist/modules/types.d.ts +149 -22
- package/dist/scripts/analyze-credit.js +54 -17
- package/dist/scripts/analyze-orders.d.ts +4 -2
- package/dist/scripts/analyze-orders.js +113 -34
- package/dist/scripts/chart_targets.js +11 -15
- package/dist/scripts/test-credit-renewal.js +12 -5
- package/docs/COW_INVARIANTS.md +71 -8
- package/docs/DEXBOT_COMPARISON.md +6 -6
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_PRICE_INVARIANT.md +1 -1
- package/docs/GRID_RECONCILE.md +18 -13
- package/docs/LOGGING.md +25 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +183 -26
- package/docs/README.md +23 -2
- package/docs/WORKFLOW.md +9 -3
- package/docs/architecture.md +68 -13
- package/docs/developer_guide.md +25 -8
- package/modules/README.md +6 -0
- package/package.json +3 -3
- package/scripts/README.md +16 -0
- package/scripts/pdev.sh +14 -1
- package/scripts/pmain.sh +23 -8
- package/scripts/ptest.sh +7 -0
- package/tests/README.md +30 -1
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
* ===============================================================================
|
|
10
10
|
*
|
|
11
11
|
* SECTION 1: EXTERNAL DEPENDENCIES
|
|
12
|
-
* SECTION 2: COW REBALANCE ENGINE
|
|
13
|
-
* SECTION 3: ORDER MANAGER CLASS
|
|
12
|
+
* SECTION 2: COW REBALANCE ENGINE - COWRebalanceEngine class (exported)
|
|
13
|
+
* SECTION 3: ORDER MANAGER CLASS - OrderManager class (exported)
|
|
14
14
|
* ===============================================================================
|
|
15
15
|
*/
|
|
16
16
|
// ===============================================================================
|
|
@@ -24,14 +24,16 @@ import AsyncLock from './async_lock.js';
|
|
|
24
24
|
import Accountant from './accounting.js';
|
|
25
25
|
import StrategyEngine from './strategy.js';
|
|
26
26
|
import SyncEngine from './sync_engine.js';
|
|
27
|
+
import { LifecycleRegistry, isSlotCreateUncertain, timingWindowMs } from './lifecycle.js';
|
|
27
28
|
import { calculateCurrentSpread, checkSpreadCondition, checkGridHealth } from './grid.js';
|
|
28
29
|
import * as Format from './format.js';
|
|
29
30
|
import { ORDER_TYPES, ORDER_STATES, REBALANCE_STATES, TIMING, LOG_LEVEL, PIPELINE_TIMING, COW_PERFORMANCE, COW_ACTIONS, GRID_LIMITS } from '../constants.js';
|
|
30
31
|
import { seedBotRuntimeConfig, createDefaultBotMetrics } from '../bot_defaults.js';
|
|
31
32
|
import { getMinOrderSize, computeChainFundTotals, hasValidAccountTotals, resolveConfigValueWithRegistry, isExplicitZeroAllocation, floatToBlockchainInt, validateBoundaryCommit, resolveGapSlots, resolveGapBand, isSlotInRail } from './utils/math.js';
|
|
32
33
|
import { validateOrder, validateGridForPersistence, validateWorkingGridFunds, checkFundDrift, reconcileGrid, optimizeRebalanceActions, projectTargetToWorkingGrid, buildStateUpdates, buildAbortedResult, buildSuccessResult, evaluateCommit } from './utils/validate.js';
|
|
33
|
-
import { resolveSpreadOrderSide, resolveLiveOrderLadderPosition, parseChainOrder, geometryTypeForSlotIndex, isOrderOnChain, resolveEdgeCount, resolveLiveEdgeAnchorPrice, compareEdge, collectRefillSlotIds,
|
|
34
|
+
import { resolveSpreadOrderSide, resolveLiveOrderLadderPosition, parseChainOrder, geometryTypeForSlotIndex, isOrderOnChain, resolveEdgeCount, resolveLiveEdgeAnchorPrice, compareEdge, collectRefillSlotIds, isFreshlyPlacedOrder, collectEdgePinnedFillIds, isStructuralRebuildStaleReason } from './utils/order.js';
|
|
34
35
|
import { parseSlotIndex } from './utils/slot.js';
|
|
36
|
+
import PlacementLock, { withPlacement as runWithPlacement } from './placement_lock.js';
|
|
35
37
|
import { getErrorMessage, getErrorCode } from '../utils/errors.js';
|
|
36
38
|
const { toFiniteNumber } = Format;
|
|
37
39
|
function stalePlacementDropReason(action, planBoundary) {
|
|
@@ -161,10 +163,29 @@ class COWRebalanceEngine {
|
|
|
161
163
|
// the whole slot's action group (a type-mismatch CANCEL is paired with
|
|
162
164
|
// a CREATE on the same id) for the grace window. The surplus is
|
|
163
165
|
// re-derived next cycle once the order has had time to prove itself.
|
|
166
|
+
//
|
|
167
|
+
// EXEMPT: a chain-observed wrong-side order. Only the correction
|
|
168
|
+
// queue's `typeMismatch` entry proves the CHAIN side is wrong; a
|
|
169
|
+
// plan-shaped CANCEL with reason 'type-mismatch' only proves
|
|
170
|
+
// master !== target, which is exactly the stale-target lag this
|
|
171
|
+
// grace exists for and must keep waiting. The pre-batch drain
|
|
172
|
+
// (correctAllPriceMismatches) normally cancels the mismatch first;
|
|
173
|
+
// when it deferred (active region) or failed, the entry is still
|
|
174
|
+
// queued — parking this action would add a second, 15-minute wait.
|
|
175
|
+
const knownWrongSide = new Set();
|
|
176
|
+
if (manager && Array.isArray(manager.ordersNeedingPriceCorrection)) {
|
|
177
|
+
for (const correction of manager.ordersNeedingPriceCorrection) {
|
|
178
|
+
if (correction?.typeMismatch === true && correction.chainOrderId) {
|
|
179
|
+
knownWrongSide.add(String(correction.chainOrderId));
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
164
183
|
const deferredFreshCancelSlots = new Set();
|
|
165
184
|
for (const action of reconcileResult.actions) {
|
|
166
185
|
if (action.type !== COW_ACTIONS.CANCEL || !action.orderId)
|
|
167
186
|
continue;
|
|
187
|
+
if (knownWrongSide.has(String(action.orderId)))
|
|
188
|
+
continue;
|
|
168
189
|
if (!manager || !isFreshlyPlacedOrder(manager, action.orderId))
|
|
169
190
|
continue;
|
|
170
191
|
const slotId = String(action.id);
|
|
@@ -321,6 +342,10 @@ class OrderManager {
|
|
|
321
342
|
sync;
|
|
322
343
|
_rebalanceState;
|
|
323
344
|
_bootstrapping;
|
|
345
|
+
_structuralRebuildDepth;
|
|
346
|
+
_structuralRebuildStartedAt;
|
|
347
|
+
_structuralRebuildDeferWarnAt;
|
|
348
|
+
_structuralRebuildReason;
|
|
324
349
|
_endOfBootstrapValidationDone;
|
|
325
350
|
_broadcastingFlag;
|
|
326
351
|
_broadcastingStartedAt;
|
|
@@ -384,6 +409,13 @@ class OrderManager {
|
|
|
384
409
|
}
|
|
385
410
|
initialSpreadCount;
|
|
386
411
|
currentSpreadCount;
|
|
412
|
+
/**
|
|
413
|
+
* True once a grid has been loaded/created and the spread metric measured.
|
|
414
|
+
* Before that both counters are 0, which is indistinguishable from a
|
|
415
|
+
* genuinely empty band — telemetry must print "unknown" instead of "0"
|
|
416
|
+
* (the restore line otherwise claims the spread band is gone).
|
|
417
|
+
*/
|
|
418
|
+
_spreadCountKnown;
|
|
387
419
|
outOfSpread;
|
|
388
420
|
assets;
|
|
389
421
|
accountId;
|
|
@@ -399,6 +431,7 @@ class OrderManager {
|
|
|
399
431
|
processedFillStore;
|
|
400
432
|
_syncLock;
|
|
401
433
|
_fillProcessingLock;
|
|
434
|
+
_placementLock;
|
|
402
435
|
_divergenceLock;
|
|
403
436
|
_gridLock;
|
|
404
437
|
_fundLock;
|
|
@@ -406,6 +439,11 @@ class OrderManager {
|
|
|
406
439
|
_pauseFundRecalc;
|
|
407
440
|
_pauseFundRecalcWatchdog;
|
|
408
441
|
_fillBatchInFlight;
|
|
442
|
+
// Consecutive blocked-cycle counter for the grid-update gate
|
|
443
|
+
// (`noteGridUpdateBlockers` in order/utils/system.ts is its only writer and
|
|
444
|
+
// reader). Cleared when the gate is evaluated healthy or not evaluated at
|
|
445
|
+
// all; a cycle count, never a timing window.
|
|
446
|
+
_gridUpdateBlockedCycles;
|
|
409
447
|
_pauseRecalcLogging;
|
|
410
448
|
_pauseRecalcLoggingWatchdog;
|
|
411
449
|
_throwOnIllegalState;
|
|
@@ -414,11 +452,42 @@ class OrderManager {
|
|
|
414
452
|
_syncGeneration;
|
|
415
453
|
_gridVersion;
|
|
416
454
|
_gridPersistenceSuspendedReason;
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
455
|
+
/**
|
|
456
|
+
* Single owner of "does this order exist on chain, and how sure are we".
|
|
457
|
+
* The legacy maps below are live views onto it — see modules/order/lifecycle.ts.
|
|
458
|
+
*/
|
|
459
|
+
_lifecycle;
|
|
460
|
+
/** Live view onto `_lifecycle`. Assigned as a whole by the COW commit path. */
|
|
461
|
+
get _committedOrderIds() { return this._lifecycle.committedIds; }
|
|
462
|
+
set _committedOrderIds(value) {
|
|
463
|
+
const ids = value ? [...value] : [];
|
|
464
|
+
this._lifecycle.markCommittedSnapshot(ids);
|
|
465
|
+
}
|
|
466
|
+
get _committedOrderIdsBuiltAt() { return this._lifecycle.committedBuiltAt; }
|
|
467
|
+
set _committedOrderIdsBuiltAt(value) { this._lifecycle.committedBuiltAt = value; }
|
|
468
|
+
get _orderIdAssignedAt() { return this._lifecycle.assignedAt; }
|
|
469
|
+
set _orderIdAssignedAt(value) {
|
|
470
|
+
this._lifecycle.assignedAt.clear();
|
|
471
|
+
if (value instanceof Map)
|
|
472
|
+
for (const [id, ts] of value)
|
|
473
|
+
this._lifecycle.assignedAt.set(id, ts);
|
|
474
|
+
}
|
|
475
|
+
get _placedAt() { return this._lifecycle.placedAt; }
|
|
476
|
+
set _placedAt(value) {
|
|
477
|
+
this._lifecycle.placedAt.clear();
|
|
478
|
+
if (value instanceof Map)
|
|
479
|
+
for (const [id, ts] of value)
|
|
480
|
+
this._lifecycle.placedAt.set(id, ts);
|
|
481
|
+
}
|
|
482
|
+
get _pendingBroadcasts() {
|
|
483
|
+
return this._lifecycle.pendingBroadcasts;
|
|
484
|
+
}
|
|
485
|
+
set _pendingBroadcasts(value) {
|
|
486
|
+
this._lifecycle.pendingBroadcasts.clear();
|
|
487
|
+
if (value instanceof Map)
|
|
488
|
+
for (const [fp, entry] of value)
|
|
489
|
+
this._lifecycle.pendingBroadcasts.set(fp, entry);
|
|
490
|
+
}
|
|
422
491
|
_gapSlots;
|
|
423
492
|
_genesis;
|
|
424
493
|
_missingGenesis;
|
|
@@ -430,6 +499,21 @@ class OrderManager {
|
|
|
430
499
|
_fundDriftLedger;
|
|
431
500
|
_pendingRecovery;
|
|
432
501
|
_pendingFillCrawls;
|
|
502
|
+
/**
|
|
503
|
+
* Chain-size resizes already absorbed by sync pass 1 (partial fill seen in
|
|
504
|
+
* an open-orders snapshot). The fill-history batch that follows the same
|
|
505
|
+
* fill must not subtract / credit the same amount a second time.
|
|
506
|
+
* Live view onto `_lifecycle`; key = chain orderId.
|
|
507
|
+
*/
|
|
508
|
+
get _chainSizeAppliedFills() {
|
|
509
|
+
return this._lifecycle.appliedFills;
|
|
510
|
+
}
|
|
511
|
+
set _chainSizeAppliedFills(value) {
|
|
512
|
+
this._lifecycle.appliedFills.clear();
|
|
513
|
+
if (value instanceof Map)
|
|
514
|
+
for (const [id, entry] of value)
|
|
515
|
+
this._lifecycle.appliedFills.set(id, entry);
|
|
516
|
+
}
|
|
433
517
|
_recentFillKeysSnapshot;
|
|
434
518
|
_lastFilledBuyPrice;
|
|
435
519
|
_lastFilledSellPrice;
|
|
@@ -473,6 +557,9 @@ class OrderManager {
|
|
|
473
557
|
* @param {Object} [config] - Configuration overrides
|
|
474
558
|
*/
|
|
475
559
|
constructor(config = {}) {
|
|
560
|
+
// The lifecycle registry MUST exist before anything below touches the
|
|
561
|
+
// legacy chain-existence structures: they are views onto it.
|
|
562
|
+
this._lifecycle = new LifecycleRegistry();
|
|
476
563
|
this.config = seedBotRuntimeConfig(config);
|
|
477
564
|
this.marketName = this.config.market || (this.config.assetA && this.config.assetB ? `${this.config.assetA}/${this.config.assetB}` : null);
|
|
478
565
|
const logFile = config.logFile || undefined;
|
|
@@ -491,6 +578,10 @@ class OrderManager {
|
|
|
491
578
|
this.sync = new SyncEngine(this);
|
|
492
579
|
this._rebalanceState = REBALANCE_STATES.NORMAL;
|
|
493
580
|
this._bootstrapping = 0;
|
|
581
|
+
this._structuralRebuildDepth = 0;
|
|
582
|
+
this._structuralRebuildStartedAt = 0;
|
|
583
|
+
this._structuralRebuildDeferWarnAt = 0;
|
|
584
|
+
this._structuralRebuildReason = null;
|
|
494
585
|
this._endOfBootstrapValidationDone = false;
|
|
495
586
|
this._broadcastingFlag = 0;
|
|
496
587
|
this._broadcastingStartedAt = 0;
|
|
@@ -514,6 +605,7 @@ class OrderManager {
|
|
|
514
605
|
this.btsBalance = { free: 0, total: 0, locked: 0 };
|
|
515
606
|
this.initialSpreadCount = 0;
|
|
516
607
|
this.currentSpreadCount = 0;
|
|
608
|
+
this._spreadCountKnown = false;
|
|
517
609
|
this.outOfSpread = 0;
|
|
518
610
|
this.assets = null;
|
|
519
611
|
this._accountTotalsPromise = null;
|
|
@@ -534,6 +626,19 @@ class OrderManager {
|
|
|
534
626
|
// Level 4: _fundLock (fund operations — innermost)
|
|
535
627
|
// Note: AsyncLock IS re-entrant (acquire detects nested calls via context).
|
|
536
628
|
this._fillProcessingLock = new AsyncLock({ timeout: TIMING.SYNC_LOCK_TIMEOUT_MS });
|
|
629
|
+
// Level 0.5: exclusive placement region shared by EVERY chain-mutating
|
|
630
|
+
// path (COW batches, startup reconcile Phase 1+2, correction drains,
|
|
631
|
+
// dust/residual cancels, pool swaps). The broadcast refcount below only
|
|
632
|
+
// defers fills; it does not stop two placement paths from broadcasting
|
|
633
|
+
// at once. See placement_lock.ts.
|
|
634
|
+
//
|
|
635
|
+
// Lock hierarchy: fillProcessingLock -> syncLock -> placement (0.5) ->
|
|
636
|
+
// gridLock -> fundLock is NOT the order — placement is acquired BEFORE
|
|
637
|
+
// _gridLock and never while holding it. The canonical order is:
|
|
638
|
+
// startBroadcasting -> placement -> gridLock -> fundLock -> release
|
|
639
|
+
// placement -> stopBroadcasting. The region-end callback wakes work
|
|
640
|
+
// that deferred on isPlacementActive().
|
|
641
|
+
this._placementLock = new PlacementLock(() => this._handlePlacementRegionEnd());
|
|
537
642
|
this._divergenceLock = new AsyncLock();
|
|
538
643
|
this._syncLock = new AsyncLock();
|
|
539
644
|
this._gridLock = new AsyncLock({
|
|
@@ -550,6 +655,7 @@ class OrderManager {
|
|
|
550
655
|
// window would report a spurious Total != Free + Committed by exactly the
|
|
551
656
|
// fills' size. The check runs once the batch settles instead.
|
|
552
657
|
this._fillBatchInFlight = 0;
|
|
658
|
+
this._gridUpdateBlockedCycles = 0;
|
|
553
659
|
this._pauseRecalcLogging = false;
|
|
554
660
|
this._pauseRecalcLoggingWatchdog = null;
|
|
555
661
|
this._throwOnIllegalState = false;
|
|
@@ -563,7 +669,7 @@ class OrderManager {
|
|
|
563
669
|
this._committedOrderIdsBuiltAt = 0;
|
|
564
670
|
this._orderIdAssignedAt = new Map();
|
|
565
671
|
// Fresh-placement timestamps (chainOrderId -> Date.now) for the
|
|
566
|
-
// surplus-cancel grace window.
|
|
672
|
+
// surplus-cancel grace window. Written by this lifecycle funnel only.
|
|
567
673
|
this._placedAt = new Map();
|
|
568
674
|
this._gapSlots = 0;
|
|
569
675
|
// Genesis (price-ladder) state. `_genesis` is established by
|
|
@@ -579,6 +685,7 @@ class OrderManager {
|
|
|
579
685
|
this._fundDriftLedger = null;
|
|
580
686
|
this._pendingRecovery = null;
|
|
581
687
|
this._pendingFillCrawls = [];
|
|
688
|
+
this._chainSizeAppliedFills = new Map();
|
|
582
689
|
this._recentFillKeysSnapshot = null;
|
|
583
690
|
this._lastStaleTotalsWarnAt = {};
|
|
584
691
|
this._lastFilledBuyPrice = null;
|
|
@@ -707,6 +814,123 @@ class OrderManager {
|
|
|
707
814
|
isBootstrapping() {
|
|
708
815
|
return this._bootstrapping > 0;
|
|
709
816
|
}
|
|
817
|
+
/**
|
|
818
|
+
* Mark a structural grid rebuild (trigger-file resync, startup regenerate)
|
|
819
|
+
* as in progress. While active, fill-driven COW rebalances DEFER: they would
|
|
820
|
+
* otherwise re-plan against the freshly rebuilt, order-id-less master and
|
|
821
|
+
* emit a create-only batch while the reconcile re-prices the live orders
|
|
822
|
+
* onto the same levels (duplicate-price orphan incident 2026-11-03).
|
|
823
|
+
* This is deliberately narrower than isBootstrapping(): the constructor's
|
|
824
|
+
* initial bootstrap is not a rebuild and fills may still rebalance there.
|
|
825
|
+
* @param {string} reason - Diagnostic label (e.g. 'trigger-file resync')
|
|
826
|
+
*/
|
|
827
|
+
beginStructuralRebuild(reason = 'structural rebuild') {
|
|
828
|
+
if (this._structuralRebuildDepth === 0)
|
|
829
|
+
this._structuralRebuildStartedAt = Date.now();
|
|
830
|
+
this._structuralRebuildDepth++;
|
|
831
|
+
this._structuralRebuildReason = reason;
|
|
832
|
+
this.logger?.log?.(`[REBUILD] Structural rebuild started (${reason}); fill-driven rebalances deferred`, 'debug');
|
|
833
|
+
}
|
|
834
|
+
/** End a structural rebuild region (balanced with beginStructuralRebuild). */
|
|
835
|
+
endStructuralRebuild() {
|
|
836
|
+
if (this._structuralRebuildDepth > 0)
|
|
837
|
+
this._structuralRebuildDepth--;
|
|
838
|
+
if (this._structuralRebuildDepth === 0) {
|
|
839
|
+
this._structuralRebuildReason = null;
|
|
840
|
+
this._structuralRebuildStartedAt = 0;
|
|
841
|
+
// Re-arm the deferred work that was parked while the rebuild owned
|
|
842
|
+
// placement. Routed through the placement-region-end handler (not
|
|
843
|
+
// the raw broadcaster) so the "never wake into a live broadcast
|
|
844
|
+
// region" rule has exactly one spelling: the rebuild's own reconcile
|
|
845
|
+
// releases the broadcast refcount before this finally runs, but a
|
|
846
|
+
// nested rebuild must not wake work into an outer live region.
|
|
847
|
+
if (this._deferredRebalanceAt)
|
|
848
|
+
this._handlePlacementRegionEnd();
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
/** Whether a structural rebuild currently owns placement. */
|
|
852
|
+
isStructuralRebuildInProgress() {
|
|
853
|
+
return this._structuralRebuildDepth > 0;
|
|
854
|
+
}
|
|
855
|
+
/** Diagnostic label of the active structural rebuild, or null. */
|
|
856
|
+
structuralRebuildReason() {
|
|
857
|
+
return this._structuralRebuildReason;
|
|
858
|
+
}
|
|
859
|
+
/**
|
|
860
|
+
* Run a chain-mutating placement path inside the exclusive region.
|
|
861
|
+
*
|
|
862
|
+
* SINGLE SOURCE OF TRUTH: every caller that can broadcast (COW, startup
|
|
863
|
+
* reconcile Phase 2, correction drain, dust/residual cancels, pool swaps)
|
|
864
|
+
* must go through this helper. It is NOT reentrant: a nested caller must not
|
|
865
|
+
* call it again — the COW body hands its lease to the pre-batch drain
|
|
866
|
+
* (`correctAllPriceMismatches({ placementLease })`), which verifies it with
|
|
867
|
+
* `isPlacementLeaseCurrent()` before mutating.
|
|
868
|
+
*
|
|
869
|
+
* Lock hierarchy: acquire placement BEFORE `_gridLock`; never acquire it
|
|
870
|
+
* while holding `_gridLock` (Level 0.5 — see the constructor's canonical
|
|
871
|
+
* lock-hierarchy block in this file).
|
|
872
|
+
*
|
|
873
|
+
* @returns `null` when the region could not be acquired; the caller must
|
|
874
|
+
* defer/abort its broadcast.
|
|
875
|
+
*/
|
|
876
|
+
async withPlacement(kind, fn, options = {}) {
|
|
877
|
+
return await runWithPlacement(this, kind, fn, options);
|
|
878
|
+
}
|
|
879
|
+
/**
|
|
880
|
+
* Acquire the exclusive placement region (see placement_lock.ts).
|
|
881
|
+
* Callers MUST release the lease in a finally block before returning.
|
|
882
|
+
* @param {string} kind - Owner label for diagnostics (e.g. 'cow-broadcast')
|
|
883
|
+
* @param {Object} [options] - Wait bound + shutdown probe
|
|
884
|
+
* @returns {Promise<Object|null>} Lease, or null on timeout/shutdown
|
|
885
|
+
*/
|
|
886
|
+
async acquirePlacement(kind, options = {}) {
|
|
887
|
+
return await this._placementLock.acquire(kind, {
|
|
888
|
+
...options,
|
|
889
|
+
isShuttingDown: options.isShuttingDown || (() => this._shuttingDown),
|
|
890
|
+
});
|
|
891
|
+
}
|
|
892
|
+
/**
|
|
893
|
+
* Non-blocking placement acquire for paths that must defer rather than queue.
|
|
894
|
+
* @param {string} kind - Owner label
|
|
895
|
+
* @returns {Object|null} Lease when free, otherwise null
|
|
896
|
+
*/
|
|
897
|
+
tryAcquirePlacement(kind) {
|
|
898
|
+
return this._placementLock.tryAcquire(kind);
|
|
899
|
+
}
|
|
900
|
+
/** Whether another placement path currently holds the exclusive region. */
|
|
901
|
+
isPlacementActive() {
|
|
902
|
+
return this._placementLock.isHeld();
|
|
903
|
+
}
|
|
904
|
+
/**
|
|
905
|
+
* Whether the given lease is STILL the one that owns the region.
|
|
906
|
+
*
|
|
907
|
+
* The authoritative nesting predicate: a nested caller (the COW pre-batch
|
|
908
|
+
* drain) must verify its inherited lease with this before mutating.
|
|
909
|
+
* `isPlacementActive()` cannot answer it — it is true for any holder, so a
|
|
910
|
+
* lease invalidated by `forceRelease()` would still read as "live".
|
|
911
|
+
* @param {Object|null|undefined} lease - Lease to verify
|
|
912
|
+
* @returns {boolean}
|
|
913
|
+
*/
|
|
914
|
+
isPlacementLeaseCurrent(lease) {
|
|
915
|
+
return this._placementLock.isLeaseCurrent(lease);
|
|
916
|
+
}
|
|
917
|
+
/** Owner label of the active placement region, or null. */
|
|
918
|
+
placementKind() {
|
|
919
|
+
return this._placementLock.kind();
|
|
920
|
+
}
|
|
921
|
+
/** Milliseconds the current placement region has been held (0 when free). */
|
|
922
|
+
placementHeldForMs() {
|
|
923
|
+
return this._placementLock.heldForMs();
|
|
924
|
+
}
|
|
925
|
+
/**
|
|
926
|
+
* True when the given staleness reason means the master grid was
|
|
927
|
+
* structurally rebuilt while a plan was in flight. Such a plan must NOT be
|
|
928
|
+
* re-planned from the fresh (order-id-less) master: a create-only re-plan
|
|
929
|
+
* duplicates every live chain order the reconcile is about to update.
|
|
930
|
+
*/
|
|
931
|
+
isStructuralRebuildStale(reason) {
|
|
932
|
+
return isStructuralRebuildStaleReason(reason);
|
|
933
|
+
}
|
|
710
934
|
/**
|
|
711
935
|
* Whether a broadcast is currently active (no side-effects).
|
|
712
936
|
* @returns {boolean}
|
|
@@ -714,12 +938,216 @@ class OrderManager {
|
|
|
714
938
|
isBroadcastingActive() {
|
|
715
939
|
return this._broadcastingFlag > 0;
|
|
716
940
|
}
|
|
941
|
+
/**
|
|
942
|
+
* Record that sync pass 1 already applied a chain-size reduction (a
|
|
943
|
+
* partial fill) for this order. The following fill-history batch checks it
|
|
944
|
+
* to stay idempotent: subtracting/crediting the same fill again shrinks the
|
|
945
|
+
* order twice and over-credits free balance (2026-11-03 log: 3.6604 ->
|
|
946
|
+
* 3.3046 by pass 1, then 3.3046 -> 2.9488 by the fill batch).
|
|
947
|
+
*
|
|
948
|
+
* Markers ACCUMULATE: two pass-1 detections before the fill queue drains
|
|
949
|
+
* (or one aggregated fill-history entry covering both) must add up, not
|
|
950
|
+
* overwrite — an overwritten marker makes the batch total unmatchable and
|
|
951
|
+
* re-opens the double-shrink. The map is mirrored into the persisted grid
|
|
952
|
+
* snapshot (debugInputs) so a restart between pass 1 and fill processing
|
|
953
|
+
* does not lose the evidence.
|
|
954
|
+
* @param {string} orderId - Chain order id
|
|
955
|
+
* @param {number} filledInt - Base-amount delta in chain integer units
|
|
956
|
+
* @returns {void}
|
|
957
|
+
*/
|
|
958
|
+
noteChainSizeAppliedFill(orderId, filledInt, maxAgeMs = TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS) {
|
|
959
|
+
const id = String(orderId || '');
|
|
960
|
+
const amount = Math.round(Number(filledInt));
|
|
961
|
+
if (!id || !Number.isFinite(amount) || amount <= 0)
|
|
962
|
+
return;
|
|
963
|
+
if (!(this._chainSizeAppliedFills instanceof Map))
|
|
964
|
+
this._chainSizeAppliedFills = new Map();
|
|
965
|
+
const now = Date.now();
|
|
966
|
+
const ttl = Number.isFinite(Number(maxAgeMs)) ? Number(maxAgeMs) : TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS;
|
|
967
|
+
const existing = this._chainSizeAppliedFills.get(id);
|
|
968
|
+
const stillFresh = existing && (now - Number(existing.at)) <= ttl;
|
|
969
|
+
this._chainSizeAppliedFills.set(id, {
|
|
970
|
+
filledInt: (stillFresh ? Number(existing.filledInt) : 0) + amount,
|
|
971
|
+
at: now,
|
|
972
|
+
});
|
|
973
|
+
// Bounded: chain-order ids turn over continuously, and this map only
|
|
974
|
+
// needs to survive the pass-1 -> fill-batch window.
|
|
975
|
+
if (this._chainSizeAppliedFills.size > 500) {
|
|
976
|
+
for (const [key, marker] of this._chainSizeAppliedFills) {
|
|
977
|
+
if (now - Number(marker.at) > ttl)
|
|
978
|
+
this._chainSizeAppliedFills.delete(key);
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Consume a pass-1 chain-size marker for a fill without requiring an exact
|
|
984
|
+
* total match. Returns the number of chain-size units this fill may treat
|
|
985
|
+
* as ALREADY reflected in `rawOnChain` (0 when there is no marker).
|
|
986
|
+
*
|
|
987
|
+
* Partial consumption is the split-batch case: pass 1 absorbed A+B, the
|
|
988
|
+
* fill queue drains A first (marker keeps B), then B. The old all-or-nothing
|
|
989
|
+
* match rejected both and double-shrunk the slot twice.
|
|
990
|
+
* @param {string} orderId - Chain order id
|
|
991
|
+
* @param {number} filledInt - Base-amount delta in chain integer units
|
|
992
|
+
* @param {number} [maxAgeMs] - Marker freshness bound
|
|
993
|
+
* @returns {number} Applied (already-in-snapshot) amount, 0..filledInt
|
|
994
|
+
*/
|
|
995
|
+
consumeChainSizeAppliedFillAmount(orderId, filledInt, maxAgeMs = TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS) {
|
|
996
|
+
const id = String(orderId || '');
|
|
997
|
+
const amount = Math.round(Number(filledInt));
|
|
998
|
+
if (!id || !Number.isFinite(amount) || amount <= 0)
|
|
999
|
+
return 0;
|
|
1000
|
+
const marker = this._chainSizeAppliedFills?.get(id);
|
|
1001
|
+
if (!marker)
|
|
1002
|
+
return 0;
|
|
1003
|
+
const ttl = Number.isFinite(Number(maxAgeMs)) ? Number(maxAgeMs) : TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS;
|
|
1004
|
+
if (Date.now() - Number(marker.at) > ttl) {
|
|
1005
|
+
this._chainSizeAppliedFills.delete(id);
|
|
1006
|
+
return 0;
|
|
1007
|
+
}
|
|
1008
|
+
const available = Math.max(0, Math.round(Number(marker.filledInt)));
|
|
1009
|
+
if (available <= 0) {
|
|
1010
|
+
this._chainSizeAppliedFills.delete(id);
|
|
1011
|
+
return 0;
|
|
1012
|
+
}
|
|
1013
|
+
const applied = Math.min(available, amount);
|
|
1014
|
+
if (available - applied > 0) {
|
|
1015
|
+
this._chainSizeAppliedFills.set(id, { filledInt: available - applied, at: Date.now() });
|
|
1016
|
+
}
|
|
1017
|
+
else {
|
|
1018
|
+
this._chainSizeAppliedFills.delete(id);
|
|
1019
|
+
}
|
|
1020
|
+
return applied;
|
|
1021
|
+
}
|
|
1022
|
+
/**
|
|
1023
|
+
* Merge persisted pass-1 markers into the in-memory set (recovery reload).
|
|
1024
|
+
* Live markers win on conflict: they may already be partially consumed,
|
|
1025
|
+
* and replacing them with the older persisted total would double-count.
|
|
1026
|
+
* Entries are sanitized and the clock is freshened (see below).
|
|
1027
|
+
* @param {unknown} payload - `{ [orderId]: { filledInt, at } }`
|
|
1028
|
+
* @returns {void}
|
|
1029
|
+
*/
|
|
1030
|
+
restoreChainSizeAppliedFills(payload) {
|
|
1031
|
+
if (!(this._chainSizeAppliedFills instanceof Map))
|
|
1032
|
+
this._chainSizeAppliedFills = new Map();
|
|
1033
|
+
if (!payload || typeof payload !== 'object')
|
|
1034
|
+
return;
|
|
1035
|
+
const now = Date.now();
|
|
1036
|
+
// A persisted marker exists only because pass 1 applied the size but
|
|
1037
|
+
// the fill history had NOT been processed when the snapshot was
|
|
1038
|
+
// written. Freshen its clock on restore: the 5-minute TTL then covers
|
|
1039
|
+
// the post-restart replay window instead of rejecting evidence simply
|
|
1040
|
+
// because the process was down longer than the live TTL.
|
|
1041
|
+
for (const [id, raw] of Object.entries(payload)) {
|
|
1042
|
+
const entry = raw;
|
|
1043
|
+
const filledInt = Math.round(Number(entry?.filledInt));
|
|
1044
|
+
const at = Number(entry?.at);
|
|
1045
|
+
if (!id || !Number.isFinite(filledInt) || filledInt <= 0 || !Number.isFinite(at))
|
|
1046
|
+
continue;
|
|
1047
|
+
if (this._chainSizeAppliedFills.has(id))
|
|
1048
|
+
continue;
|
|
1049
|
+
this._chainSizeAppliedFills.set(id, { filledInt, at: now });
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
/** Snapshot the live pass-1 markers for persistence (empty when none). */
|
|
1053
|
+
snapshotChainSizeAppliedFills() {
|
|
1054
|
+
const out = {};
|
|
1055
|
+
if (!(this._chainSizeAppliedFills instanceof Map))
|
|
1056
|
+
return out;
|
|
1057
|
+
const now = Date.now();
|
|
1058
|
+
const ttl = timingWindowMs('CHAIN_SIZE_APPLIED_MARKER_TTL_MS', 5 * 60 * 1000);
|
|
1059
|
+
for (const [id, entry] of this._chainSizeAppliedFills) {
|
|
1060
|
+
const filledInt = Math.round(Number(entry?.filledInt));
|
|
1061
|
+
const at = Number(entry?.at);
|
|
1062
|
+
if (!id || !Number.isFinite(filledInt) || filledInt <= 0 || !Number.isFinite(at))
|
|
1063
|
+
continue;
|
|
1064
|
+
if (now - at > ttl)
|
|
1065
|
+
continue;
|
|
1066
|
+
out[id] = { filledInt, at };
|
|
1067
|
+
}
|
|
1068
|
+
return out;
|
|
1069
|
+
}
|
|
1070
|
+
/**
|
|
1071
|
+
* Wake the tasks that deferred on isPlacementActive() when the region ends.
|
|
1072
|
+
* The broadcast-region-end listener reschedules a deferred rebalance and
|
|
1073
|
+
* drains queued fills; without this a fill deferred because a correction
|
|
1074
|
+
* drain/COW held placement would only retry on the next unrelated event.
|
|
1075
|
+
* Skipped while a broadcast region is still active (stopBroadcasting will
|
|
1076
|
+
* fire the same listeners when it ends).
|
|
1077
|
+
* @returns {void}
|
|
1078
|
+
*/
|
|
1079
|
+
_handlePlacementRegionEnd() {
|
|
1080
|
+
if (this._broadcastingFlag > 0)
|
|
1081
|
+
return;
|
|
1082
|
+
this._fireBroadcastRegionEnd();
|
|
1083
|
+
}
|
|
717
1084
|
/**
|
|
718
1085
|
* @param {boolean} value
|
|
719
1086
|
* @returns {void}
|
|
720
1087
|
*/
|
|
721
1088
|
setShuttingDown(value) {
|
|
722
1089
|
this._shuttingDown = value === true;
|
|
1090
|
+
if (this._shuttingDown) {
|
|
1091
|
+
// Never let a hung placement region block shutdown/exit, and force
|
|
1092
|
+
// every waiter to abandon its batch instead of broadcasting.
|
|
1093
|
+
this._placementLock.forceRelease();
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
/**
|
|
1097
|
+
* Auto-clears a stale placement region after
|
|
1098
|
+
* TIMING.PLACEMENT_STALE_CLEAR_MS so a hung chain call / missed release
|
|
1099
|
+
* cannot park fills and rebalances forever. The holder's lease becomes a
|
|
1100
|
+
* no-op on release (token guard), so a late finally cannot release a
|
|
1101
|
+
* successor.
|
|
1102
|
+
*
|
|
1103
|
+
* TWO GUARDS, ONE TEARDOWN:
|
|
1104
|
+
* - A bot-level batch/recovery still in flight and YOUNGER than the bound is
|
|
1105
|
+
* evidence of progress, so the region is not torn down (a long create-group
|
|
1106
|
+
* reconcile must survive). Only a batch older than the same bound is
|
|
1107
|
+
* treated as part of the hung region.
|
|
1108
|
+
* - When the teardown does happen it also clears a broadcast refcount that
|
|
1109
|
+
* is itself past its bound, atomically. A refcount raised by a placement
|
|
1110
|
+
* holder belongs to the same region (the COW body and the reconcile call
|
|
1111
|
+
* startBroadcasting() INSIDE it), so clearing both in one place stops the
|
|
1112
|
+
* maintenance tick from tearing down placement first and the refcount
|
|
1113
|
+
* second (two authorities, one hung region). A refcount raised OUTSIDE any
|
|
1114
|
+
* region (the spread-correction pre-broadcast hold in `order/grid.ts`) is
|
|
1115
|
+
* not this region's and is left to `_clearStaleBroadcastFlag`, which still
|
|
1116
|
+
* refuses to act on its own while a placement lease is held.
|
|
1117
|
+
* @param {number} [maxAgeMs] - Override for tests
|
|
1118
|
+
* @param {Object} [options] - `batchInFlightMs`: age of the LAST raised
|
|
1119
|
+
* bot-level pipeline counter (0/undefined when none).
|
|
1120
|
+
* @returns {boolean} true when a stale region was force-released
|
|
1121
|
+
*/
|
|
1122
|
+
_clearStalePlacementRegion(maxAgeMs = TIMING.PLACEMENT_STALE_CLEAR_MS, options = {}) {
|
|
1123
|
+
if (!this.isPlacementActive())
|
|
1124
|
+
return false;
|
|
1125
|
+
const heldMs = this.placementHeldForMs();
|
|
1126
|
+
const limit = Number(maxAgeMs) > 0 ? Number(maxAgeMs) : TIMING.PLACEMENT_STALE_CLEAR_MS;
|
|
1127
|
+
if (heldMs <= limit)
|
|
1128
|
+
return false;
|
|
1129
|
+
const batchInFlightMs = Number(options?.batchInFlightMs);
|
|
1130
|
+
if (Number.isFinite(batchInFlightMs) && batchInFlightMs > 0 && batchInFlightMs <= limit) {
|
|
1131
|
+
this.logger?.log?.(`[PLACEMENT] Watchdog skipped: region held by ${this.placementKind() || 'unknown'} for ${Math.round(heldMs / 1000)}s, ` +
|
|
1132
|
+
`but the pipeline is still making progress (in flight ${Math.round(batchInFlightMs / 1000)}s)`, 'debug');
|
|
1133
|
+
return false;
|
|
1134
|
+
}
|
|
1135
|
+
const kind = this.placementKind() || 'unknown';
|
|
1136
|
+
this.logger?.log?.(`[PLACEMENT] Force-releasing stale placement region held by ${kind} for ${Math.round(heldMs / 1000)}s ` +
|
|
1137
|
+
`(watchdog ${Math.round(limit / 1000)}s); fills/rebalances must not stay parked`, 'error');
|
|
1138
|
+
this._placementLock.forceRelease();
|
|
1139
|
+
// Same hung region: drop its broadcast refcount too (see JSDoc).
|
|
1140
|
+
if (this._broadcastingFlag > 0 && this._broadcastingStartedAt > 0) {
|
|
1141
|
+
const broadcastAgeMs = Date.now() - this._broadcastingStartedAt;
|
|
1142
|
+
if (broadcastAgeMs > TIMING.BROADCAST_STALE_CLEAR_MS) {
|
|
1143
|
+
this.logger?.log?.(`[PLACEMENT] Also clearing the stale broadcast region (${Math.round(broadcastAgeMs / 1000)}s) it held; ` +
|
|
1144
|
+
`one teardown for one hung region`, 'warn');
|
|
1145
|
+
this._broadcastingFlag = 0;
|
|
1146
|
+
this._broadcastingStartedAt = 0;
|
|
1147
|
+
this._fireBroadcastRegionEnd();
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
return true;
|
|
723
1151
|
}
|
|
724
1152
|
/**
|
|
725
1153
|
* Auto-clears a stale broadcast flag after 120s so a hung
|
|
@@ -733,6 +1161,14 @@ class OrderManager {
|
|
|
733
1161
|
if (this._broadcastingFlag > 0 && this._broadcastingStartedAt > 0) {
|
|
734
1162
|
const elapsed = Date.now() - this._broadcastingStartedAt;
|
|
735
1163
|
if (elapsed > TIMING.BROADCAST_STALE_CLEAR_MS) {
|
|
1164
|
+
// A legitimately long reconcile/COW can hold placement well past
|
|
1165
|
+
// the watchdog while its refcount is still valid. Never cut the
|
|
1166
|
+
// region out from under a live placement holder: the refcount
|
|
1167
|
+
// is the leak signal only when nobody owns the region.
|
|
1168
|
+
if (this.isPlacementActive()) {
|
|
1169
|
+
this.logger?.log?.(`[BROADCAST] Stale-flag watchdog skipped: placement region still held by ${this.placementKind() || 'unknown'} for ${Math.round(this.placementHeldForMs() / 1000)}s`, 'debug');
|
|
1170
|
+
return;
|
|
1171
|
+
}
|
|
736
1172
|
this.logger?.log?.(`[BROADCAST] Auto-clearing stale broadcast flag after ${Math.round(TIMING.BROADCAST_STALE_CLEAR_MS / 1000)}s`, 'warn');
|
|
737
1173
|
// Hard-reset to 0 (not decrement) — this is a safety valve for
|
|
738
1174
|
// a hung broadcast where stopBroadcasting() was never called.
|
|
@@ -1188,7 +1624,7 @@ class OrderManager {
|
|
|
1188
1624
|
}
|
|
1189
1625
|
}
|
|
1190
1626
|
/**
|
|
1191
|
-
* @param {Array<import('
|
|
1627
|
+
* @param {Array<import('../types.js').Order>} orders
|
|
1192
1628
|
* @param {Object} info
|
|
1193
1629
|
* @returns {Promise<any>}
|
|
1194
1630
|
*/
|
|
@@ -1290,6 +1726,42 @@ class OrderManager {
|
|
|
1290
1726
|
}
|
|
1291
1727
|
}
|
|
1292
1728
|
}
|
|
1729
|
+
/**
|
|
1730
|
+
* The single answer to "does this order exist on chain, and how sure are we".
|
|
1731
|
+
*
|
|
1732
|
+
* Every consumer that used to infer this from one of the legacy maps must
|
|
1733
|
+
* read it here instead — that is the whole point of the lifecycle registry:
|
|
1734
|
+
* four sites consulting four different structures is how the engine ended up
|
|
1735
|
+
* with four different answers for the same order.
|
|
1736
|
+
*
|
|
1737
|
+
* A `createUncertain` slot flag (persisted across restarts) is folded into
|
|
1738
|
+
* `phase === 'unknown'` here, so no caller has to remember to check it.
|
|
1739
|
+
*
|
|
1740
|
+
* @param identity - Slot id, chain order id, or pending-broadcast fingerprint
|
|
1741
|
+
* @returns {LifecycleView} Unified read surface (never null)
|
|
1742
|
+
*/
|
|
1743
|
+
getLifecycle(identity) {
|
|
1744
|
+
return this._lifecycle.view(identity);
|
|
1745
|
+
}
|
|
1746
|
+
/**
|
|
1747
|
+
* Lifecycle view for a slot, folding in the slot's own persisted flags.
|
|
1748
|
+
* Prefer this over `getLifecycle(slotId)` wherever a slot object is at hand.
|
|
1749
|
+
* @param slot - Grid slot (ManagedOrder) or null
|
|
1750
|
+
* @returns {LifecycleView}
|
|
1751
|
+
*/
|
|
1752
|
+
getSlotLifecycle(slot) {
|
|
1753
|
+
if (!slot)
|
|
1754
|
+
return this._lifecycle.view(null);
|
|
1755
|
+
const identity = slot.orderId || slot.id || null;
|
|
1756
|
+
if (isSlotCreateUncertain(slot)) {
|
|
1757
|
+
// The persisted marker outranks an older registry phase: a CREATE
|
|
1758
|
+
// whose receipt was lost must never read as "nothing there".
|
|
1759
|
+
// Read-only: `withPhase` reports `unknown` without recording it, so a
|
|
1760
|
+
// read cannot keep the unresolved bound alive by refreshing its clock.
|
|
1761
|
+
return this._lifecycle.withPhase(identity, 'unknown', 'broadcast-receipt');
|
|
1762
|
+
}
|
|
1763
|
+
return this._lifecycle.view(identity);
|
|
1764
|
+
}
|
|
1293
1765
|
_normalizeOrderUpdateOptions(options = {}) {
|
|
1294
1766
|
if (options === null || typeof options !== 'object' || Array.isArray(options)) {
|
|
1295
1767
|
throw new TypeError('Order update options must be an object');
|
|
@@ -1363,41 +1835,11 @@ class OrderManager {
|
|
|
1363
1835
|
}
|
|
1364
1836
|
const updatedOrder = deepFreeze({ ...nextOrder });
|
|
1365
1837
|
const id = order.id;
|
|
1366
|
-
//
|
|
1367
|
-
//
|
|
1368
|
-
//
|
|
1369
|
-
//
|
|
1370
|
-
|
|
1371
|
-
// younger than SYNC_LOCK_TIMEOUT_MS.
|
|
1372
|
-
if (updatedOrder.orderId && (!oldOrder?.orderId || oldOrder.orderId !== updatedOrder.orderId)) {
|
|
1373
|
-
// Prune stamps older than the guard window — they can never be
|
|
1374
|
-
// consulted again (the consumer defers only while the stamp is
|
|
1375
|
-
// younger than SYNC_LOCK_TIMEOUT_MS), so keeping them would leak one
|
|
1376
|
-
// entry per orderId on a perpetual-turnover bot.
|
|
1377
|
-
for (const [orderId, assignedAt] of this._orderIdAssignedAt) {
|
|
1378
|
-
if (Date.now() - assignedAt >= TIMING.SYNC_LOCK_TIMEOUT_MS) {
|
|
1379
|
-
this._orderIdAssignedAt.delete(orderId);
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
this._orderIdAssignedAt.set(updatedOrder.orderId, Date.now());
|
|
1383
|
-
}
|
|
1384
|
-
// Fresh-placement timestamp for the surplus-cancel grace window: a
|
|
1385
|
-
// slot gaining an orderId it did not have is a new placement (create,
|
|
1386
|
-
// adopt, rotation target), and surplus sweeps must not cancel it
|
|
1387
|
-
// before it has had time to prove itself. Same-id updates do not
|
|
1388
|
-
// refresh the stamp, so grace cannot extend forever. Snapshot/grid
|
|
1389
|
-
// load paths are excluded: reloaded orders are old by definition, and
|
|
1390
|
-
// stamping them would neuter surplus sweeps for 15 minutes after every
|
|
1391
|
-
// restart.
|
|
1392
|
-
try {
|
|
1393
|
-
const prevId = oldOrder?.orderId != null ? String(oldOrder.orderId) : '';
|
|
1394
|
-
const nextId = updatedOrder.orderId != null ? String(updatedOrder.orderId) : '';
|
|
1395
|
-
const isLoadContext = context === 'grid-load' || context === 'grid-init';
|
|
1396
|
-
if (nextId && nextId !== prevId && !isLoadContext) {
|
|
1397
|
-
recordOrderPlacement(this, nextId);
|
|
1398
|
-
}
|
|
1399
|
-
}
|
|
1400
|
-
catch { /* bookkeeping must never break order updates */ }
|
|
1838
|
+
// Project the mutation into the lifecycle registry. This is the ONLY
|
|
1839
|
+
// writer of the chain-existence facts (phase, assignedAt, placedAt): the
|
|
1840
|
+
// stamps used to be written a second time right here, which meant two
|
|
1841
|
+
// copies of every clock and an O(records) prune loop per order update.
|
|
1842
|
+
this._syncLifecycleFromMutation(oldOrder, updatedOrder, context);
|
|
1401
1843
|
const newMap = cloneMap(this.orders);
|
|
1402
1844
|
newMap.set(id, updatedOrder);
|
|
1403
1845
|
this.orders = Object.freeze(newMap);
|
|
@@ -1409,6 +1851,99 @@ class OrderManager {
|
|
|
1409
1851
|
this._markGridDirty();
|
|
1410
1852
|
return true;
|
|
1411
1853
|
}
|
|
1854
|
+
/**
|
|
1855
|
+
* Project one order mutation into the lifecycle registry.
|
|
1856
|
+
*
|
|
1857
|
+
* This is the single write funnel for chain existence: every mutation goes
|
|
1858
|
+
* through `_applyOrderUpdate`, so no call site has to remember which of the
|
|
1859
|
+
* seven legacy structures to touch. A slot is the record's identity, so a
|
|
1860
|
+
* chain-id change (rotation / re-place) updates the same record instead of
|
|
1861
|
+
* starting a parallel one.
|
|
1862
|
+
*
|
|
1863
|
+
* @param previous - Slot before the mutation (absent on create)
|
|
1864
|
+
* @param next - Slot after the mutation
|
|
1865
|
+
* @param context - Mutation context label
|
|
1866
|
+
* @returns {void}
|
|
1867
|
+
*/
|
|
1868
|
+
_syncLifecycleFromMutation(previous, next, context) {
|
|
1869
|
+
try {
|
|
1870
|
+
const slotId = String(next.id);
|
|
1871
|
+
const nextId = next.orderId != null && String(next.orderId).length > 0 ? String(next.orderId) : '';
|
|
1872
|
+
const prevId = previous?.orderId != null && String(previous.orderId).length > 0 ? String(previous.orderId) : '';
|
|
1873
|
+
// Persisted "the CREATE result was lost" marker outranks everything:
|
|
1874
|
+
// the order may be live, so the phase must never read as absent.
|
|
1875
|
+
if (isSlotCreateUncertain(next)) {
|
|
1876
|
+
this._lifecycle.markUnknown(nextId || null, { slotId, evidence: 'broadcast-receipt' });
|
|
1877
|
+
return;
|
|
1878
|
+
}
|
|
1879
|
+
if (!nextId) {
|
|
1880
|
+
// The slot holds no chain id. Marking it `intent` would claim
|
|
1881
|
+
// "nothing is out there", which we may only say when nothing
|
|
1882
|
+
// unresolved is on record — unless this very mutation is the
|
|
1883
|
+
// explicit clear of the create-uncertain marker (the load-time
|
|
1884
|
+
// sanitizer does exactly that after neutralising the slot).
|
|
1885
|
+
const clearedUncertainMarker = isSlotCreateUncertain(previous) && !isSlotCreateUncertain(next);
|
|
1886
|
+
if (clearedUncertainMarker) {
|
|
1887
|
+
this._lifecycle.markIntent({ slotId });
|
|
1888
|
+
}
|
|
1889
|
+
else {
|
|
1890
|
+
this._lifecycle.markIntentIfResolved(slotId);
|
|
1891
|
+
}
|
|
1892
|
+
return;
|
|
1893
|
+
}
|
|
1894
|
+
if (nextId !== prevId) {
|
|
1895
|
+
// A freshly assigned chain id (create/adopt broadcast in flight, or
|
|
1896
|
+
// not yet visible to a lagging read) must not read as "absent from
|
|
1897
|
+
// chain": absence decisions defer while it is younger than
|
|
1898
|
+
// SYNC_LOCK_TIMEOUT_MS (isRecentlyAssigned).
|
|
1899
|
+
this._lifecycle.markAssigned(slotId, nextId);
|
|
1900
|
+
// Fresh-placement timestamp for the surplus-cancel grace window: a
|
|
1901
|
+
// slot gaining an orderId it did not have is a new placement
|
|
1902
|
+
// (create, adopt, rotation target), and surplus sweeps must not
|
|
1903
|
+
// cancel it before it has had time to prove itself. Same-id updates
|
|
1904
|
+
// do not refresh the stamp, so grace cannot extend forever.
|
|
1905
|
+
// Snapshot/grid load paths are excluded: reloaded orders are old by
|
|
1906
|
+
// definition, and stamping them would neuter surplus sweeps for 15
|
|
1907
|
+
// minutes after every restart.
|
|
1908
|
+
const isLoadContext = context === 'grid-load' || context === 'grid-init';
|
|
1909
|
+
if (!isLoadContext)
|
|
1910
|
+
this._lifecycle.markPlaced(nextId, undefined, slotId);
|
|
1911
|
+
}
|
|
1912
|
+
// This funnel only knows the LOCAL slot state, not where that state
|
|
1913
|
+
// came from, so the evidence is 'local': it must not advance
|
|
1914
|
+
// `observedAt` ("when did the CHAIN last tell us"). The chain-backed
|
|
1915
|
+
// sites (sync pass 1 match, pass 2 adoption) call markLanded with
|
|
1916
|
+
// 'chain-snapshot' / 'adoption' themselves.
|
|
1917
|
+
if (next.state === ORDER_STATES.PARTIAL) {
|
|
1918
|
+
this._lifecycle.observe(nextId, { slotId, phase: 'filling', evidence: 'local' });
|
|
1919
|
+
}
|
|
1920
|
+
else if (next.state === ORDER_STATES.ACTIVE) {
|
|
1921
|
+
this._lifecycle.observe(nextId, { slotId, phase: 'landed', evidence: 'local' });
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
catch { /* lifecycle bookkeeping must never break an order update */ }
|
|
1925
|
+
}
|
|
1926
|
+
/**
|
|
1927
|
+
* Abandon an unresolved pending CREATE (TTL sweep / give-up).
|
|
1928
|
+
*
|
|
1929
|
+
* Distinct from `_pendingBroadcasts.delete()`: dropping an entry because the
|
|
1930
|
+
* create RESOLVED leaves the slot free, while dropping it because we ran out
|
|
1931
|
+
* of patience means the order may still be live. Collapsing both into one
|
|
1932
|
+
* delete is how a lost receipt turned into a clean hole that got re-CREATEd.
|
|
1933
|
+
*
|
|
1934
|
+
* @param fingerprint - Pending-broadcast fingerprint being abandoned
|
|
1935
|
+
* @param entry - The abandoned entry (carries the slot id)
|
|
1936
|
+
* @returns {void}
|
|
1937
|
+
*/
|
|
1938
|
+
_expirePendingBroadcast(fingerprint, entry) {
|
|
1939
|
+
const slotId = entry?.slotId != null ? String(entry.slotId) : null;
|
|
1940
|
+
try {
|
|
1941
|
+
this._lifecycle.deletePending(fingerprint);
|
|
1942
|
+
if (slotId)
|
|
1943
|
+
this._lifecycle.markUnknown(null, { slotId, evidence: 'local' });
|
|
1944
|
+
}
|
|
1945
|
+
catch { /* bookkeeping only */ }
|
|
1946
|
+
}
|
|
1412
1947
|
_peekWorkingGrid() {
|
|
1413
1948
|
const stack = this._currentWorkingGridStack;
|
|
1414
1949
|
return stack.length > 0 ? stack[stack.length - 1] : null;
|
|
@@ -1419,7 +1954,16 @@ class OrderManager {
|
|
|
1419
1954
|
return;
|
|
1420
1955
|
}
|
|
1421
1956
|
try {
|
|
1422
|
-
|
|
1957
|
+
// A mutation during a structural rebuild is ALWAYS structural, even
|
|
1958
|
+
// when its own context (`sync-pass1-partial`, `sync-pass2-orphan`,
|
|
1959
|
+
// `fill-place`) looks like a fill. The rebuild's fresh master has no
|
|
1960
|
+
// live chain ids, so a create-only re-plan at these levels duplicates
|
|
1961
|
+
// the reconcile's update-first placement. Stamping the reason here
|
|
1962
|
+
// makes the classifier independent of the individual sync context.
|
|
1963
|
+
const reason = this.isStructuralRebuildInProgress()
|
|
1964
|
+
? `master mutation during structural rebuild (${context})`
|
|
1965
|
+
: `master mutation during ${(this._rebalanceState || '').toLowerCase()} (${context})`;
|
|
1966
|
+
wg.markStale(reason);
|
|
1423
1967
|
wg.syncFromMaster(this.orders, orderId, this._gridVersion);
|
|
1424
1968
|
}
|
|
1425
1969
|
catch (syncErr) {
|
|
@@ -1428,7 +1972,7 @@ class OrderManager {
|
|
|
1428
1972
|
}
|
|
1429
1973
|
}
|
|
1430
1974
|
/**
|
|
1431
|
-
* @param {Array<import('
|
|
1975
|
+
* @param {Array<import('../types.js').Order>} updates - Order updates to apply
|
|
1432
1976
|
* @param {string} [context] - Update context label
|
|
1433
1977
|
* @param {unknown} [options]
|
|
1434
1978
|
* @returns {Promise<boolean>}
|
|
@@ -1517,7 +2061,7 @@ class OrderManager {
|
|
|
1517
2061
|
return result || { isValid: true };
|
|
1518
2062
|
}
|
|
1519
2063
|
/**
|
|
1520
|
-
* @param {Array<import('
|
|
2064
|
+
* @param {Array<import('../types.js').Order>} orders - Filled orders
|
|
1521
2065
|
* @param {Set<string>} [excl] - Order IDs to exclude
|
|
1522
2066
|
* @param {Object} [options]
|
|
1523
2067
|
* @returns {Promise<any>}
|
|
@@ -1535,6 +2079,13 @@ class OrderManager {
|
|
|
1535
2079
|
// instead of waiting in-lock; processFillsOnly already recorded
|
|
1536
2080
|
// the boundary crawls, so a later derivation applies them. The
|
|
1537
2081
|
// region-end hook schedules the deferred rebalance.
|
|
2082
|
+
//
|
|
2083
|
+
// RE-ANCHOR BUDGET SNAPSHOT: the absolute fill anchor is a per-batch
|
|
2084
|
+
// budget spent at PLAN time (strategy.calculateTargetGrid). Carry the
|
|
2085
|
+
// pre-plan value so a stale-plan re-plan can hand back exactly what
|
|
2086
|
+
// the abandoned plan spent (a static batch base would wrongly refund
|
|
2087
|
+
// an anchor an earlier chunk legitimately committed).
|
|
2088
|
+
const reanchorBudgetBefore = Number(this._boundaryReanchorBudget);
|
|
1538
2089
|
const rebalanceResult = await this.performSafeRebalance(orders, excl, {
|
|
1539
2090
|
...(options || {}),
|
|
1540
2091
|
deferIfBroadcasting: true
|
|
@@ -1544,6 +2095,7 @@ class OrderManager {
|
|
|
1544
2095
|
// stale between planning and broadcast.
|
|
1545
2096
|
rebalanceResult.fills = orders;
|
|
1546
2097
|
rebalanceResult.excludeIds = excl || new Set();
|
|
2098
|
+
rebalanceResult._reanchorBudgetBeforePlan = reanchorBudgetBefore;
|
|
1547
2099
|
return rebalanceResult;
|
|
1548
2100
|
}
|
|
1549
2101
|
const workingGrid = new WorkingGrid(this.orders, { baseVersion: this._gridVersion });
|
|
@@ -1558,7 +2110,7 @@ class OrderManager {
|
|
|
1558
2110
|
};
|
|
1559
2111
|
}
|
|
1560
2112
|
/**
|
|
1561
|
-
* @returns {Array<import('
|
|
2113
|
+
* @returns {Array<import('../types.js').Order>}
|
|
1562
2114
|
*/
|
|
1563
2115
|
getInitialOrdersToActivate() {
|
|
1564
2116
|
// Apply coreOrders limit from config
|
|
@@ -1711,19 +2263,16 @@ class OrderManager {
|
|
|
1711
2263
|
* Also used when the boundary is (re)anchored outside the fill path (grid
|
|
1712
2264
|
* reset / recenter / rejected-snapshot rebuild): those records belong to
|
|
1713
2265
|
* the previous generation, and applying their relative deltas onto a fresh
|
|
1714
|
-
* anchor would double-count movement the anchor already contains.
|
|
2266
|
+
* anchor would double-count movement the anchor already contains. That is
|
|
2267
|
+
* the explicit allowed-loss path: the grid generation is being discarded
|
|
2268
|
+
* (rebuild, rejected snapshot, restart replay), so the relative deltas no
|
|
2269
|
+
* longer have a geometry to apply onto.
|
|
1715
2270
|
*
|
|
1716
2271
|
* Marks the grid dirty so the cleared array reaches disk even on paths
|
|
1717
2272
|
* that do not otherwise persist.
|
|
1718
2273
|
*
|
|
1719
2274
|
* @param {string} reason - Log/debug label for the drop
|
|
1720
|
-
|
|
1721
|
-
/**
|
|
1722
|
-
* Drop ALL owed fill-crawl records.
|
|
1723
|
-
*
|
|
1724
|
-
* This is the explicit allowed-loss path: the grid generation is being
|
|
1725
|
-
* discarded (rebuild, rejected snapshot, restart replay), so the relative
|
|
1726
|
-
* deltas no longer have a geometry to apply onto.
|
|
2275
|
+
* @param {string} detail - Extra context appended to the drop log line
|
|
1727
2276
|
*/
|
|
1728
2277
|
_clearPendingFillCrawls(reason = 'unspecified', detail = '') {
|
|
1729
2278
|
if (!Array.isArray(this._pendingFillCrawls) || this._pendingFillCrawls.length === 0)
|
|
@@ -1824,8 +2373,23 @@ class OrderManager {
|
|
|
1824
2373
|
const boundary = newIdx == null ? this.boundaryIdx : newIdx;
|
|
1825
2374
|
if (boundary == null)
|
|
1826
2375
|
return;
|
|
1827
|
-
|
|
2376
|
+
// Geometry: resolveGapSlots is the single source of truth
|
|
2377
|
+
// (_gapSlots ?? config-derived). Reading the raw field with a `0`
|
|
2378
|
+
// fallback printed `gapSlots=0 sellStart=boundary+1` for every
|
|
2379
|
+
// restore that happens before the loader publishes _gapSlots —
|
|
2380
|
+
// indistinguishable in the log from a genuinely collapsed spread
|
|
2381
|
+
// band (field report: "bot lost its fixed spread, only one slot is
|
|
2382
|
+
// used"). A config-derived gap is always a valid geometry; an
|
|
2383
|
+
// absent one is unknown, never zero.
|
|
2384
|
+
const gapSlots = resolveGapSlots(this);
|
|
1828
2385
|
const sellStart = boundary + 1 + gapSlots;
|
|
2386
|
+
// The spread metric only exists once a grid has been loaded or
|
|
2387
|
+
// created (both counters are 0 on a fresh manager, which is
|
|
2388
|
+
// indistinguishable from a genuinely empty band). Printing 0
|
|
2389
|
+
// before the measurement claims an empty band we have not measured.
|
|
2390
|
+
const spreadCount = this._spreadCountKnown
|
|
2391
|
+
? (this.currentSpreadCount ?? this.initialSpreadCount)
|
|
2392
|
+
: '?';
|
|
1829
2393
|
const countActive = (type) => {
|
|
1830
2394
|
let n = 0;
|
|
1831
2395
|
for (const o of this.orders.values()) {
|
|
@@ -1835,7 +2399,7 @@ class OrderManager {
|
|
|
1835
2399
|
return n;
|
|
1836
2400
|
};
|
|
1837
2401
|
this.logger?.log?.(`[BOUNDARY] ${source} boundary=${boundary} sellStart=${sellStart} gapSlots=${gapSlots} ` +
|
|
1838
|
-
`spreadCount=${
|
|
2402
|
+
`spreadCount=${spreadCount} ` +
|
|
1839
2403
|
`activeBuy=${countActive(ORDER_TYPES.BUY)} activeSell=${countActive(ORDER_TYPES.SELL)}`, 'debug');
|
|
1840
2404
|
}
|
|
1841
2405
|
catch {
|
|
@@ -2159,7 +2723,7 @@ class OrderManager {
|
|
|
2159
2723
|
return true;
|
|
2160
2724
|
}
|
|
2161
2725
|
/**
|
|
2162
|
-
* @param {Map<string, import('
|
|
2726
|
+
* @param {Map<string, import('../types.js').Order>} targetGrid
|
|
2163
2727
|
* @param {number} targetBoundary
|
|
2164
2728
|
* @returns {Object}
|
|
2165
2729
|
*/
|
|
@@ -2341,6 +2905,38 @@ class OrderManager {
|
|
|
2341
2905
|
// _fillProcessingLock (the lock's acquisition timeout is 20s; the
|
|
2342
2906
|
// wait is 30s). A fill-driven rebalance defers instead, leaving
|
|
2343
2907
|
// the crawls recorded by processFillsOnly for a later derivation.
|
|
2908
|
+
//
|
|
2909
|
+
// STRUCTURAL REBUILD DEFER (2026-11-03 live duplicate-order incident): a
|
|
2910
|
+
// trigger-file rebuild rebuilds + reconciles the grid. A
|
|
2911
|
+
// fill-driven COW that plans during that window re-plans against
|
|
2912
|
+
// the freshly rebuilt (order-id-less) master and emits a
|
|
2913
|
+
// create-only plan, while startup reconcile re-prices the live
|
|
2914
|
+
// orders onto the same levels — both generations land as
|
|
2915
|
+
// duplicate-price orphans. The reconcile owns placement until
|
|
2916
|
+
// endStructuralRebuild(); fills stay queued and are drained by the
|
|
2917
|
+
// bootstrap consumer. This check must run before
|
|
2918
|
+
// deferIfBroadcasting because the fill path passes that flag.
|
|
2919
|
+
if (this.isStructuralRebuildInProgress()) {
|
|
2920
|
+
const rebuildAgeMs = this._structuralRebuildStartedAt > 0
|
|
2921
|
+
? Date.now() - this._structuralRebuildStartedAt
|
|
2922
|
+
: 0;
|
|
2923
|
+
const deferBoundMs = timingWindowMs('STRUCTURAL_REBUILD_DEFER_MAX_MS', 3 * 60 * 1000);
|
|
2924
|
+
const overBound = rebuildAgeMs > deferBoundMs;
|
|
2925
|
+
const lastWarnAt = Number(this._structuralRebuildDeferWarnAt) || 0;
|
|
2926
|
+
const shouldWarn = overBound && Date.now() - lastWarnAt > 60000;
|
|
2927
|
+
if (shouldWarn) {
|
|
2928
|
+
this._structuralRebuildDeferWarnAt = Date.now();
|
|
2929
|
+
}
|
|
2930
|
+
this.logger.log(`[SAFE-REBALANCE] Deferred: structural rebuild in progress (${this.structuralRebuildReason() || 'rebuild'}, ` +
|
|
2931
|
+
`${Math.round(rebuildAgeMs / 1000)}s); startup reconcile owns placement`, shouldWarn ? 'warn' : (overBound ? 'debug' : 'info'));
|
|
2932
|
+
this._deferredRebalanceAt = Date.now();
|
|
2933
|
+
return { ...buildAbortedResult('structural-rebuild-deferred'), deferred: true };
|
|
2934
|
+
}
|
|
2935
|
+
if (this.isPlacementActive()) {
|
|
2936
|
+
this.logger.log(`[SAFE-REBALANCE] Deferred: placement region held by ${this.placementKind() || 'another path'}; rebalance retries after it ends`, 'info');
|
|
2937
|
+
this._deferredRebalanceAt = Date.now();
|
|
2938
|
+
return { ...buildAbortedResult('placement-active-deferred'), deferred: true };
|
|
2939
|
+
}
|
|
2344
2940
|
if (options?.deferIfBroadcasting && this.isBroadcastingActive()) {
|
|
2345
2941
|
this.logger.log('[SAFE-REBALANCE] Deferred: broadcast/placement activity active; rebalance retries after the region ends', 'info');
|
|
2346
2942
|
this._deferredRebalanceAt = Date.now();
|
|
@@ -2413,6 +3009,12 @@ class OrderManager {
|
|
|
2413
3009
|
// delay the cancel-only evacuation queue. Queued-once per slot; the
|
|
2414
3010
|
// corrections runner (correctAllPriceMismatches) executes the cancel
|
|
2415
3011
|
// and settles the slot back to a spread placeholder.
|
|
3012
|
+
//
|
|
3013
|
+
// The COW pipeline holds a queued cancel back from its pre-batch drain
|
|
3014
|
+
// when the same batch ROTATES that chain order (the rotation IS the
|
|
3015
|
+
// evacuation) — see updateOrdersOnChainBatchCOWBody. Queueing both is
|
|
3016
|
+
// therefore safe: the rotation runs, and the held entry is dropped as
|
|
3017
|
+
// stale on the next drain because the slot no longer owns the order.
|
|
2416
3018
|
try {
|
|
2417
3019
|
this._processGapEvacuationTeeth(result?.evacReady);
|
|
2418
3020
|
}
|
|
@@ -2537,6 +3139,21 @@ class OrderManager {
|
|
|
2537
3139
|
deepFreeze(order);
|
|
2538
3140
|
}
|
|
2539
3141
|
}
|
|
3142
|
+
// LIFE-CYCLE PROJECTION OF THE WHOLE-MAP SWAP.
|
|
3143
|
+
// The COW commit replaces `this.orders` wholesale instead of
|
|
3144
|
+
// calling _applyOrderUpdate per slot, so it used to bypass the
|
|
3145
|
+
// single write funnel entirely: an order the batch just PLACED got
|
|
3146
|
+
// no `assignedAt` and — worse — no `placedAt`, leaving the 15-minute
|
|
3147
|
+
// surplus-cancel grace inert for the primary placement path (a
|
|
3148
|
+
// competing sweep could cancel a seconds-old order). Projecting each
|
|
3149
|
+
// slot through the same funnel closes that hole; the funnel only
|
|
3150
|
+
// records bookkeeping, it does not change what is committed.
|
|
3151
|
+
// Runs BEFORE the swap so each projection sees the previous slot.
|
|
3152
|
+
for (const [slotId, nextOrder] of finalMap.entries()) {
|
|
3153
|
+
if (!nextOrder)
|
|
3154
|
+
continue;
|
|
3155
|
+
this._syncLifecycleFromMutation(this.orders.get(slotId), nextOrder, 'cow-commit');
|
|
3156
|
+
}
|
|
2540
3157
|
this.orders = Object.freeze(finalMap);
|
|
2541
3158
|
this._setBoundary(commitBoundary);
|
|
2542
3159
|
// Pending-crawl bookkeeping: an accepted non-null commit
|
|
@@ -2566,16 +3183,15 @@ class OrderManager {
|
|
|
2566
3183
|
this._gridVersion++;
|
|
2567
3184
|
committed = true;
|
|
2568
3185
|
// Track orderIds from successful COW commits for recovery sync protection.
|
|
2569
|
-
//
|
|
2570
|
-
//
|
|
2571
|
-
//
|
|
3186
|
+
// ONE publish: the snapshot setter stamps every id AND moves the
|
|
3187
|
+
// built-at window, so a second `_committedOrderIdsBuiltAt` write
|
|
3188
|
+
// would only re-stamp the same ids with a later clock.
|
|
2572
3189
|
const newCommittedIds = new Set();
|
|
2573
3190
|
for (const [, order] of finalMap.entries()) {
|
|
2574
3191
|
if (order.orderId)
|
|
2575
3192
|
newCommittedIds.add(order.orderId);
|
|
2576
3193
|
}
|
|
2577
3194
|
this._committedOrderIds = newCommittedIds;
|
|
2578
|
-
this._committedOrderIdsBuiltAt = Date.now();
|
|
2579
3195
|
// Index caches invalidated by _gridVersion bump above — lazy getters recompute from this.orders
|
|
2580
3196
|
});
|
|
2581
3197
|
if (!committed) {
|