dexbot 1.7.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +3 -3
- package/analysis/README.md +33 -13
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/account_resolver.js +5 -6
- package/dist/analysis/analyze_liquidity.js +15 -4
- package/dist/analysis/bot_key_utils.d.ts +44 -1
- package/dist/analysis/bot_key_utils.js +76 -7
- package/dist/analysis/fills_cache.js +2 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -19
- package/dist/analysis/grid_correction_check.js +12 -30
- package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
- package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
- package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
- package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
- package/dist/analysis/liquidity/liquidity_view.js +11 -0
- package/dist/analysis/resolve_bot_accounts.js +7 -6
- package/dist/analysis/trade_profitability.js +2 -1
- package/dist/credential-daemon.js +13 -2
- package/dist/dexbot.js +4 -4
- package/dist/modules/account_bots.d.ts +11 -1
- package/dist/modules/account_orders.d.ts +12 -0
- package/dist/modules/account_orders.js +30 -0
- package/dist/modules/bitshares-native/transport.js +24 -6
- package/dist/modules/bitshares_client.d.ts +24 -6
- package/dist/modules/bitshares_client.js +44 -7
- package/dist/modules/bot_settings.d.ts +2 -0
- package/dist/modules/constants.d.ts +26 -3
- package/dist/modules/constants.js +63 -8
- package/dist/modules/dexbot_class.d.ts +1 -6
- package/dist/modules/dexbot_class.js +14 -0
- package/dist/modules/dexbot_cow_runtime.d.ts +97 -7
- package/dist/modules/dexbot_cow_runtime.js +620 -132
- package/dist/modules/dexbot_fill_runtime.js +124 -35
- package/dist/modules/dexbot_maintenance_runtime.js +226 -73
- package/dist/modules/dexbot_startup_runtime.js +32 -20
- package/dist/modules/dexbot_state_recovery.d.ts +33 -1
- package/dist/modules/dexbot_state_recovery.js +167 -5
- package/dist/modules/node_connect_policy.d.ts +8 -1
- package/dist/modules/node_connect_policy.js +18 -2
- package/dist/modules/node_flap_score.d.ts +87 -0
- package/dist/modules/node_flap_score.js +124 -0
- package/dist/modules/node_health_cache.d.ts +40 -3
- package/dist/modules/node_health_cache.js +66 -20
- package/dist/modules/node_manager.d.ts +101 -19
- package/dist/modules/node_manager.js +255 -65
- package/dist/modules/order/accounting.js +56 -7
- package/dist/modules/order/format.d.ts +24 -10
- package/dist/modules/order/format.js +30 -10
- package/dist/modules/order/grid.js +82 -11
- package/dist/modules/order/grid_reconcile.d.ts +5 -1
- package/dist/modules/order/grid_reconcile.js +333 -210
- package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
- package/dist/modules/order/grid_reconcile_internal.js +207 -87
- package/dist/modules/order/index.d.ts +4 -0
- package/dist/modules/order/lifecycle.d.ts +353 -0
- package/dist/modules/order/lifecycle.js +1109 -0
- package/dist/modules/order/manager.d.ts +271 -5
- package/dist/modules/order/manager.js +651 -48
- package/dist/modules/order/placement_lock.d.ts +102 -0
- package/dist/modules/order/placement_lock.js +222 -0
- package/dist/modules/order/strategy.js +13 -1
- package/dist/modules/order/sync_engine.d.ts +13 -1
- package/dist/modules/order/sync_engine.js +153 -53
- package/dist/modules/order/utils/math.d.ts +22 -1
- package/dist/modules/order/utils/math.js +54 -1
- package/dist/modules/order/utils/order.d.ts +75 -5
- package/dist/modules/order/utils/order.js +158 -9
- package/dist/modules/order/utils/system.d.ts +18 -0
- package/dist/modules/order/utils/system.js +84 -8
- package/dist/modules/order/utils/validate.d.ts +7 -1
- package/dist/modules/order/utils/validate.js +48 -2
- package/dist/modules/outcome.d.ts +54 -0
- package/dist/modules/outcome.js +51 -0
- package/dist/modules/settings_merge.d.ts +4 -0
- package/dist/modules/settings_merge.js +9 -0
- package/dist/modules/timing_policy.d.ts +82 -0
- package/dist/modules/timing_policy.js +244 -0
- package/dist/modules/types.d.ts +132 -0
- package/dist/scripts/analyze-credit.js +54 -17
- package/dist/scripts/analyze-orders.d.ts +4 -2
- package/dist/scripts/analyze-orders.js +113 -34
- package/dist/scripts/chart_targets.js +11 -15
- package/dist/scripts/test-credit-renewal.js +12 -5
- package/docs/COW_INVARIANTS.md +67 -8
- package/docs/DEXBOT_COMPARISON.md +6 -6
- package/docs/EVOLUTION.md +9 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_PRICE_INVARIANT.md +1 -1
- package/docs/GRID_RECONCILE.md +17 -12
- package/docs/LOGGING.md +25 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +175 -26
- package/docs/README.md +23 -2
- package/docs/WORKFLOW.md +9 -3
- package/docs/architecture.md +42 -5
- package/docs/developer_guide.md +25 -8
- package/modules/README.md +3 -0
- package/package.json +1 -1
- package/scripts/README.md +16 -0
- package/scripts/pdev.sh +14 -1
- package/scripts/pmain.sh +23 -8
- package/scripts/ptest.sh +7 -0
- package/tests/README.md +30 -1
|
@@ -24,14 +24,16 @@ import AsyncLock from './async_lock.js';
|
|
|
24
24
|
import Accountant from './accounting.js';
|
|
25
25
|
import StrategyEngine from './strategy.js';
|
|
26
26
|
import SyncEngine from './sync_engine.js';
|
|
27
|
+
import { LifecycleRegistry, isSlotCreateUncertain, timingWindowMs } from './lifecycle.js';
|
|
27
28
|
import { calculateCurrentSpread, checkSpreadCondition, checkGridHealth } from './grid.js';
|
|
28
29
|
import * as Format from './format.js';
|
|
29
30
|
import { ORDER_TYPES, ORDER_STATES, REBALANCE_STATES, TIMING, LOG_LEVEL, PIPELINE_TIMING, COW_PERFORMANCE, COW_ACTIONS, GRID_LIMITS } from '../constants.js';
|
|
30
31
|
import { seedBotRuntimeConfig, createDefaultBotMetrics } from '../bot_defaults.js';
|
|
31
32
|
import { getMinOrderSize, computeChainFundTotals, hasValidAccountTotals, resolveConfigValueWithRegistry, isExplicitZeroAllocation, floatToBlockchainInt, validateBoundaryCommit, resolveGapSlots, resolveGapBand, isSlotInRail } from './utils/math.js';
|
|
32
33
|
import { validateOrder, validateGridForPersistence, validateWorkingGridFunds, checkFundDrift, reconcileGrid, optimizeRebalanceActions, projectTargetToWorkingGrid, buildStateUpdates, buildAbortedResult, buildSuccessResult, evaluateCommit } from './utils/validate.js';
|
|
33
|
-
import { resolveSpreadOrderSide, resolveLiveOrderLadderPosition, parseChainOrder, geometryTypeForSlotIndex, isOrderOnChain, resolveEdgeCount, resolveLiveEdgeAnchorPrice, compareEdge, collectRefillSlotIds,
|
|
34
|
+
import { resolveSpreadOrderSide, resolveLiveOrderLadderPosition, parseChainOrder, geometryTypeForSlotIndex, isOrderOnChain, resolveEdgeCount, resolveLiveEdgeAnchorPrice, compareEdge, collectRefillSlotIds, isFreshlyPlacedOrder, collectEdgePinnedFillIds, isStructuralRebuildStaleReason } from './utils/order.js';
|
|
34
35
|
import { parseSlotIndex } from './utils/slot.js';
|
|
36
|
+
import PlacementLock, { withPlacement as runWithPlacement } from './placement_lock.js';
|
|
35
37
|
import { getErrorMessage, getErrorCode } from '../utils/errors.js';
|
|
36
38
|
const { toFiniteNumber } = Format;
|
|
37
39
|
function stalePlacementDropReason(action, planBoundary) {
|
|
@@ -321,6 +323,10 @@ class OrderManager {
|
|
|
321
323
|
sync;
|
|
322
324
|
_rebalanceState;
|
|
323
325
|
_bootstrapping;
|
|
326
|
+
_structuralRebuildDepth;
|
|
327
|
+
_structuralRebuildStartedAt;
|
|
328
|
+
_structuralRebuildDeferWarnAt;
|
|
329
|
+
_structuralRebuildReason;
|
|
324
330
|
_endOfBootstrapValidationDone;
|
|
325
331
|
_broadcastingFlag;
|
|
326
332
|
_broadcastingStartedAt;
|
|
@@ -384,6 +390,13 @@ class OrderManager {
|
|
|
384
390
|
}
|
|
385
391
|
initialSpreadCount;
|
|
386
392
|
currentSpreadCount;
|
|
393
|
+
/**
|
|
394
|
+
* True once a grid has been loaded/created and the spread metric measured.
|
|
395
|
+
* Before that both counters are 0, which is indistinguishable from a
|
|
396
|
+
* genuinely empty band — telemetry must print "unknown" instead of "0"
|
|
397
|
+
* (the restore line otherwise claims the spread band is gone).
|
|
398
|
+
*/
|
|
399
|
+
_spreadCountKnown;
|
|
387
400
|
outOfSpread;
|
|
388
401
|
assets;
|
|
389
402
|
accountId;
|
|
@@ -399,6 +412,7 @@ class OrderManager {
|
|
|
399
412
|
processedFillStore;
|
|
400
413
|
_syncLock;
|
|
401
414
|
_fillProcessingLock;
|
|
415
|
+
_placementLock;
|
|
402
416
|
_divergenceLock;
|
|
403
417
|
_gridLock;
|
|
404
418
|
_fundLock;
|
|
@@ -414,11 +428,42 @@ class OrderManager {
|
|
|
414
428
|
_syncGeneration;
|
|
415
429
|
_gridVersion;
|
|
416
430
|
_gridPersistenceSuspendedReason;
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
431
|
+
/**
|
|
432
|
+
* Single owner of "does this order exist on chain, and how sure are we".
|
|
433
|
+
* The legacy maps below are live views onto it — see modules/order/lifecycle.ts.
|
|
434
|
+
*/
|
|
435
|
+
_lifecycle;
|
|
436
|
+
/** Live view onto `_lifecycle`. Assigned as a whole by the COW commit path. */
|
|
437
|
+
get _committedOrderIds() { return this._lifecycle.committedIds; }
|
|
438
|
+
set _committedOrderIds(value) {
|
|
439
|
+
const ids = value ? [...value] : [];
|
|
440
|
+
this._lifecycle.markCommittedSnapshot(ids);
|
|
441
|
+
}
|
|
442
|
+
get _committedOrderIdsBuiltAt() { return this._lifecycle.committedBuiltAt; }
|
|
443
|
+
set _committedOrderIdsBuiltAt(value) { this._lifecycle.committedBuiltAt = value; }
|
|
444
|
+
get _orderIdAssignedAt() { return this._lifecycle.assignedAt; }
|
|
445
|
+
set _orderIdAssignedAt(value) {
|
|
446
|
+
this._lifecycle.assignedAt.clear();
|
|
447
|
+
if (value instanceof Map)
|
|
448
|
+
for (const [id, ts] of value)
|
|
449
|
+
this._lifecycle.assignedAt.set(id, ts);
|
|
450
|
+
}
|
|
451
|
+
get _placedAt() { return this._lifecycle.placedAt; }
|
|
452
|
+
set _placedAt(value) {
|
|
453
|
+
this._lifecycle.placedAt.clear();
|
|
454
|
+
if (value instanceof Map)
|
|
455
|
+
for (const [id, ts] of value)
|
|
456
|
+
this._lifecycle.placedAt.set(id, ts);
|
|
457
|
+
}
|
|
458
|
+
get _pendingBroadcasts() {
|
|
459
|
+
return this._lifecycle.pendingBroadcasts;
|
|
460
|
+
}
|
|
461
|
+
set _pendingBroadcasts(value) {
|
|
462
|
+
this._lifecycle.pendingBroadcasts.clear();
|
|
463
|
+
if (value instanceof Map)
|
|
464
|
+
for (const [fp, entry] of value)
|
|
465
|
+
this._lifecycle.pendingBroadcasts.set(fp, entry);
|
|
466
|
+
}
|
|
422
467
|
_gapSlots;
|
|
423
468
|
_genesis;
|
|
424
469
|
_missingGenesis;
|
|
@@ -430,6 +475,21 @@ class OrderManager {
|
|
|
430
475
|
_fundDriftLedger;
|
|
431
476
|
_pendingRecovery;
|
|
432
477
|
_pendingFillCrawls;
|
|
478
|
+
/**
|
|
479
|
+
* Chain-size resizes already absorbed by sync pass 1 (partial fill seen in
|
|
480
|
+
* an open-orders snapshot). The fill-history batch that follows the same
|
|
481
|
+
* fill must not subtract / credit the same amount a second time.
|
|
482
|
+
* Live view onto `_lifecycle`; key = chain orderId.
|
|
483
|
+
*/
|
|
484
|
+
get _chainSizeAppliedFills() {
|
|
485
|
+
return this._lifecycle.appliedFills;
|
|
486
|
+
}
|
|
487
|
+
set _chainSizeAppliedFills(value) {
|
|
488
|
+
this._lifecycle.appliedFills.clear();
|
|
489
|
+
if (value instanceof Map)
|
|
490
|
+
for (const [id, entry] of value)
|
|
491
|
+
this._lifecycle.appliedFills.set(id, entry);
|
|
492
|
+
}
|
|
433
493
|
_recentFillKeysSnapshot;
|
|
434
494
|
_lastFilledBuyPrice;
|
|
435
495
|
_lastFilledSellPrice;
|
|
@@ -473,6 +533,9 @@ class OrderManager {
|
|
|
473
533
|
* @param {Object} [config] - Configuration overrides
|
|
474
534
|
*/
|
|
475
535
|
constructor(config = {}) {
|
|
536
|
+
// The lifecycle registry MUST exist before anything below touches the
|
|
537
|
+
// legacy chain-existence structures: they are views onto it.
|
|
538
|
+
this._lifecycle = new LifecycleRegistry();
|
|
476
539
|
this.config = seedBotRuntimeConfig(config);
|
|
477
540
|
this.marketName = this.config.market || (this.config.assetA && this.config.assetB ? `${this.config.assetA}/${this.config.assetB}` : null);
|
|
478
541
|
const logFile = config.logFile || undefined;
|
|
@@ -491,6 +554,10 @@ class OrderManager {
|
|
|
491
554
|
this.sync = new SyncEngine(this);
|
|
492
555
|
this._rebalanceState = REBALANCE_STATES.NORMAL;
|
|
493
556
|
this._bootstrapping = 0;
|
|
557
|
+
this._structuralRebuildDepth = 0;
|
|
558
|
+
this._structuralRebuildStartedAt = 0;
|
|
559
|
+
this._structuralRebuildDeferWarnAt = 0;
|
|
560
|
+
this._structuralRebuildReason = null;
|
|
494
561
|
this._endOfBootstrapValidationDone = false;
|
|
495
562
|
this._broadcastingFlag = 0;
|
|
496
563
|
this._broadcastingStartedAt = 0;
|
|
@@ -514,6 +581,7 @@ class OrderManager {
|
|
|
514
581
|
this.btsBalance = { free: 0, total: 0, locked: 0 };
|
|
515
582
|
this.initialSpreadCount = 0;
|
|
516
583
|
this.currentSpreadCount = 0;
|
|
584
|
+
this._spreadCountKnown = false;
|
|
517
585
|
this.outOfSpread = 0;
|
|
518
586
|
this.assets = null;
|
|
519
587
|
this._accountTotalsPromise = null;
|
|
@@ -534,6 +602,19 @@ class OrderManager {
|
|
|
534
602
|
// Level 4: _fundLock (fund operations — innermost)
|
|
535
603
|
// Note: AsyncLock IS re-entrant (acquire detects nested calls via context).
|
|
536
604
|
this._fillProcessingLock = new AsyncLock({ timeout: TIMING.SYNC_LOCK_TIMEOUT_MS });
|
|
605
|
+
// Level 0.5: exclusive placement region shared by EVERY chain-mutating
|
|
606
|
+
// path (COW batches, startup reconcile Phase 1+2, correction drains,
|
|
607
|
+
// dust/residual cancels, pool swaps). The broadcast refcount below only
|
|
608
|
+
// defers fills; it does not stop two placement paths from broadcasting
|
|
609
|
+
// at once. See placement_lock.ts.
|
|
610
|
+
//
|
|
611
|
+
// Lock hierarchy: fillProcessingLock -> syncLock -> placement (0.5) ->
|
|
612
|
+
// gridLock -> fundLock is NOT the order — placement is acquired BEFORE
|
|
613
|
+
// _gridLock and never while holding it. The canonical order is:
|
|
614
|
+
// startBroadcasting -> placement -> gridLock -> fundLock -> release
|
|
615
|
+
// placement -> stopBroadcasting. The region-end callback wakes work
|
|
616
|
+
// that deferred on isPlacementActive().
|
|
617
|
+
this._placementLock = new PlacementLock(() => this._handlePlacementRegionEnd());
|
|
537
618
|
this._divergenceLock = new AsyncLock();
|
|
538
619
|
this._syncLock = new AsyncLock();
|
|
539
620
|
this._gridLock = new AsyncLock({
|
|
@@ -579,6 +660,7 @@ class OrderManager {
|
|
|
579
660
|
this._fundDriftLedger = null;
|
|
580
661
|
this._pendingRecovery = null;
|
|
581
662
|
this._pendingFillCrawls = [];
|
|
663
|
+
this._chainSizeAppliedFills = new Map();
|
|
582
664
|
this._recentFillKeysSnapshot = null;
|
|
583
665
|
this._lastStaleTotalsWarnAt = {};
|
|
584
666
|
this._lastFilledBuyPrice = null;
|
|
@@ -707,6 +789,122 @@ class OrderManager {
|
|
|
707
789
|
isBootstrapping() {
|
|
708
790
|
return this._bootstrapping > 0;
|
|
709
791
|
}
|
|
792
|
+
/**
|
|
793
|
+
* Mark a structural grid rebuild (trigger-file resync, startup regenerate)
|
|
794
|
+
* as in progress. While active, fill-driven COW rebalances DEFER: they would
|
|
795
|
+
* otherwise re-plan against the freshly rebuilt, order-id-less master and
|
|
796
|
+
* emit a create-only batch while the reconcile re-prices the live orders
|
|
797
|
+
* onto the same levels (duplicate-price orphan incident 2026-11-03).
|
|
798
|
+
* This is deliberately narrower than isBootstrapping(): the constructor's
|
|
799
|
+
* initial bootstrap is not a rebuild and fills may still rebalance there.
|
|
800
|
+
* @param {string} reason - Diagnostic label (e.g. 'trigger-file resync')
|
|
801
|
+
*/
|
|
802
|
+
beginStructuralRebuild(reason = 'structural rebuild') {
|
|
803
|
+
if (this._structuralRebuildDepth === 0)
|
|
804
|
+
this._structuralRebuildStartedAt = Date.now();
|
|
805
|
+
this._structuralRebuildDepth++;
|
|
806
|
+
this._structuralRebuildReason = reason;
|
|
807
|
+
this.logger?.log?.(`[REBUILD] Structural rebuild started (${reason}); fill-driven rebalances deferred`, 'debug');
|
|
808
|
+
}
|
|
809
|
+
/** End a structural rebuild region (balanced with beginStructuralRebuild). */
|
|
810
|
+
endStructuralRebuild() {
|
|
811
|
+
if (this._structuralRebuildDepth > 0)
|
|
812
|
+
this._structuralRebuildDepth--;
|
|
813
|
+
if (this._structuralRebuildDepth === 0) {
|
|
814
|
+
this._structuralRebuildReason = null;
|
|
815
|
+
this._structuralRebuildStartedAt = 0;
|
|
816
|
+
// Re-arm the deferred work that was parked while the rebuild owned
|
|
817
|
+
// placement. Routed through the placement-region-end handler (not
|
|
818
|
+
// the raw broadcaster) so the "never wake into a live broadcast
|
|
819
|
+
// region" rule has exactly one spelling: the rebuild's own reconcile
|
|
820
|
+
// releases the broadcast refcount before this finally runs, but a
|
|
821
|
+
// nested rebuild must not wake work into an outer live region.
|
|
822
|
+
if (this._deferredRebalanceAt)
|
|
823
|
+
this._handlePlacementRegionEnd();
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
/** Whether a structural rebuild currently owns placement. */
|
|
827
|
+
isStructuralRebuildInProgress() {
|
|
828
|
+
return this._structuralRebuildDepth > 0;
|
|
829
|
+
}
|
|
830
|
+
/** Diagnostic label of the active structural rebuild, or null. */
|
|
831
|
+
structuralRebuildReason() {
|
|
832
|
+
return this._structuralRebuildReason;
|
|
833
|
+
}
|
|
834
|
+
/**
|
|
835
|
+
* Run a chain-mutating placement path inside the exclusive region.
|
|
836
|
+
*
|
|
837
|
+
* SINGLE SOURCE OF TRUTH: every caller that can broadcast (COW, startup
|
|
838
|
+
* reconcile Phase 2, correction drain, dust/residual cancels, pool swaps)
|
|
839
|
+
* must go through this helper. It is NOT reentrant: a nested caller must not
|
|
840
|
+
* call it again — the COW body hands its lease to the pre-batch drain
|
|
841
|
+
* (`correctAllPriceMismatches({ placementLease })`), which verifies it with
|
|
842
|
+
* `isPlacementLeaseCurrent()` before mutating.
|
|
843
|
+
*
|
|
844
|
+
* Lock hierarchy: acquire placement BEFORE `_gridLock`; never acquire it
|
|
845
|
+
* while holding `_gridLock` (Level 0.5 — see `manageLocks`).
|
|
846
|
+
*
|
|
847
|
+
* @returns `null` when the region could not be acquired; the caller must
|
|
848
|
+
* defer/abort its broadcast.
|
|
849
|
+
*/
|
|
850
|
+
async withPlacement(kind, fn, options = {}) {
|
|
851
|
+
return await runWithPlacement(this, kind, fn, options);
|
|
852
|
+
}
|
|
853
|
+
/**
|
|
854
|
+
* Acquire the exclusive placement region (see placement_lock.ts).
|
|
855
|
+
* Callers MUST release the lease in a finally block before returning.
|
|
856
|
+
* @param {string} kind - Owner label for diagnostics (e.g. 'cow-broadcast')
|
|
857
|
+
* @param {Object} [options] - Wait bound + shutdown probe
|
|
858
|
+
* @returns {Promise<Object|null>} Lease, or null on timeout/shutdown
|
|
859
|
+
*/
|
|
860
|
+
async acquirePlacement(kind, options = {}) {
|
|
861
|
+
return await this._placementLock.acquire(kind, {
|
|
862
|
+
...options,
|
|
863
|
+
isShuttingDown: options.isShuttingDown || (() => this._shuttingDown),
|
|
864
|
+
});
|
|
865
|
+
}
|
|
866
|
+
/**
|
|
867
|
+
* Non-blocking placement acquire for paths that must defer rather than queue.
|
|
868
|
+
* @param {string} kind - Owner label
|
|
869
|
+
* @returns {Object|null} Lease when free, otherwise null
|
|
870
|
+
*/
|
|
871
|
+
tryAcquirePlacement(kind) {
|
|
872
|
+
return this._placementLock.tryAcquire(kind);
|
|
873
|
+
}
|
|
874
|
+
/** Whether another placement path currently holds the exclusive region. */
|
|
875
|
+
isPlacementActive() {
|
|
876
|
+
return this._placementLock.isHeld();
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* Whether the given lease is STILL the one that owns the region.
|
|
880
|
+
*
|
|
881
|
+
* The authoritative nesting predicate: a nested caller (the COW pre-batch
|
|
882
|
+
* drain) must verify its inherited lease with this before mutating.
|
|
883
|
+
* `isPlacementActive()` cannot answer it — it is true for any holder, so a
|
|
884
|
+
* lease invalidated by `forceRelease()` would still read as "live".
|
|
885
|
+
* @param {Object|null|undefined} lease - Lease to verify
|
|
886
|
+
* @returns {boolean}
|
|
887
|
+
*/
|
|
888
|
+
isPlacementLeaseCurrent(lease) {
|
|
889
|
+
return this._placementLock.isLeaseCurrent(lease);
|
|
890
|
+
}
|
|
891
|
+
/** Owner label of the active placement region, or null. */
|
|
892
|
+
placementKind() {
|
|
893
|
+
return this._placementLock.kind();
|
|
894
|
+
}
|
|
895
|
+
/** Milliseconds the current placement region has been held (0 when free). */
|
|
896
|
+
placementHeldForMs() {
|
|
897
|
+
return this._placementLock.heldForMs();
|
|
898
|
+
}
|
|
899
|
+
/**
|
|
900
|
+
* True when the given staleness reason means the master grid was
|
|
901
|
+
* structurally rebuilt while a plan was in flight. Such a plan must NOT be
|
|
902
|
+
* re-planned from the fresh (order-id-less) master: a create-only re-plan
|
|
903
|
+
* duplicates every live chain order the reconcile is about to update.
|
|
904
|
+
*/
|
|
905
|
+
isStructuralRebuildStale(reason) {
|
|
906
|
+
return isStructuralRebuildStaleReason(reason);
|
|
907
|
+
}
|
|
710
908
|
/**
|
|
711
909
|
* Whether a broadcast is currently active (no side-effects).
|
|
712
910
|
* @returns {boolean}
|
|
@@ -714,12 +912,226 @@ class OrderManager {
|
|
|
714
912
|
isBroadcastingActive() {
|
|
715
913
|
return this._broadcastingFlag > 0;
|
|
716
914
|
}
|
|
915
|
+
/**
|
|
916
|
+
* Record that sync pass 1 already applied a chain-size reduction (a
|
|
917
|
+
* partial fill) for this order. The following fill-history batch checks it
|
|
918
|
+
* to stay idempotent: subtracting/crediting the same fill again shrinks the
|
|
919
|
+
* order twice and over-credits free balance (2026-11-03 log: 3.6604 ->
|
|
920
|
+
* 3.3046 by pass 1, then 3.3046 -> 2.9488 by the fill batch).
|
|
921
|
+
*
|
|
922
|
+
* Markers ACCUMULATE: two pass-1 detections before the fill queue drains
|
|
923
|
+
* (or one aggregated fill-history entry covering both) must add up, not
|
|
924
|
+
* overwrite — an overwritten marker makes the batch total unmatchable and
|
|
925
|
+
* re-opens the double-shrink. The map is mirrored into the persisted grid
|
|
926
|
+
* snapshot (debugInputs) so a restart between pass 1 and fill processing
|
|
927
|
+
* does not lose the evidence.
|
|
928
|
+
* @param {string} orderId - Chain order id
|
|
929
|
+
* @param {number} filledInt - Base-amount delta in chain integer units
|
|
930
|
+
* @returns {void}
|
|
931
|
+
*/
|
|
932
|
+
noteChainSizeAppliedFill(orderId, filledInt, maxAgeMs = TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS) {
|
|
933
|
+
const id = String(orderId || '');
|
|
934
|
+
const amount = Math.round(Number(filledInt));
|
|
935
|
+
if (!id || !Number.isFinite(amount) || amount <= 0)
|
|
936
|
+
return;
|
|
937
|
+
if (!(this._chainSizeAppliedFills instanceof Map))
|
|
938
|
+
this._chainSizeAppliedFills = new Map();
|
|
939
|
+
const now = Date.now();
|
|
940
|
+
const ttl = Number.isFinite(Number(maxAgeMs)) ? Number(maxAgeMs) : TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS;
|
|
941
|
+
const existing = this._chainSizeAppliedFills.get(id);
|
|
942
|
+
const stillFresh = existing && (now - Number(existing.at)) <= ttl;
|
|
943
|
+
this._chainSizeAppliedFills.set(id, {
|
|
944
|
+
filledInt: (stillFresh ? Number(existing.filledInt) : 0) + amount,
|
|
945
|
+
at: now,
|
|
946
|
+
});
|
|
947
|
+
// Bounded: chain-order ids turn over continuously, and this map only
|
|
948
|
+
// needs to survive the pass-1 -> fill-batch window.
|
|
949
|
+
if (this._chainSizeAppliedFills.size > 500) {
|
|
950
|
+
for (const [key, marker] of this._chainSizeAppliedFills) {
|
|
951
|
+
if (now - Number(marker.at) > ttl)
|
|
952
|
+
this._chainSizeAppliedFills.delete(key);
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
/**
|
|
957
|
+
* Consume a pass-1 chain-size marker for a fill without requiring an exact
|
|
958
|
+
* total match. Returns the number of chain-size units this fill may treat
|
|
959
|
+
* as ALREADY reflected in `rawOnChain` (0 when there is no marker).
|
|
960
|
+
*
|
|
961
|
+
* Partial consumption is the split-batch case: pass 1 absorbed A+B, the
|
|
962
|
+
* fill queue drains A first (marker keeps B), then B. The old all-or-nothing
|
|
963
|
+
* match rejected both and double-shrunk the slot twice.
|
|
964
|
+
* @param {string} orderId - Chain order id
|
|
965
|
+
* @param {number} filledInt - Base-amount delta in chain integer units
|
|
966
|
+
* @param {number} [maxAgeMs] - Marker freshness bound
|
|
967
|
+
* @returns {number} Applied (already-in-snapshot) amount, 0..filledInt
|
|
968
|
+
*/
|
|
969
|
+
consumeChainSizeAppliedFillAmount(orderId, filledInt, maxAgeMs = TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS) {
|
|
970
|
+
const id = String(orderId || '');
|
|
971
|
+
const amount = Math.round(Number(filledInt));
|
|
972
|
+
if (!id || !Number.isFinite(amount) || amount <= 0)
|
|
973
|
+
return 0;
|
|
974
|
+
const marker = this._chainSizeAppliedFills?.get(id);
|
|
975
|
+
if (!marker)
|
|
976
|
+
return 0;
|
|
977
|
+
const ttl = Number.isFinite(Number(maxAgeMs)) ? Number(maxAgeMs) : TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS;
|
|
978
|
+
if (Date.now() - Number(marker.at) > ttl) {
|
|
979
|
+
this._chainSizeAppliedFills.delete(id);
|
|
980
|
+
return 0;
|
|
981
|
+
}
|
|
982
|
+
const available = Math.max(0, Math.round(Number(marker.filledInt)));
|
|
983
|
+
if (available <= 0) {
|
|
984
|
+
this._chainSizeAppliedFills.delete(id);
|
|
985
|
+
return 0;
|
|
986
|
+
}
|
|
987
|
+
const applied = Math.min(available, amount);
|
|
988
|
+
if (available - applied > 0) {
|
|
989
|
+
this._chainSizeAppliedFills.set(id, { filledInt: available - applied, at: Date.now() });
|
|
990
|
+
}
|
|
991
|
+
else {
|
|
992
|
+
this._chainSizeAppliedFills.delete(id);
|
|
993
|
+
}
|
|
994
|
+
return applied;
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Boolean compatibility wrapper for {@link consumeChainSizeAppliedFillAmount}.
|
|
998
|
+
* @param {string} orderId - Chain order id
|
|
999
|
+
* @param {number} filledInt - Base-amount delta in chain integer units
|
|
1000
|
+
* @param {number} [maxAgeMs] - Marker freshness bound
|
|
1001
|
+
* @returns {boolean} true when any part of the fill was already applied
|
|
1002
|
+
*/
|
|
1003
|
+
consumeChainSizeAppliedFill(orderId, filledInt, maxAgeMs = TIMING.CHAIN_SIZE_APPLIED_MARKER_TTL_MS) {
|
|
1004
|
+
return this.consumeChainSizeAppliedFillAmount(orderId, filledInt, maxAgeMs) > 0;
|
|
1005
|
+
}
|
|
1006
|
+
/**
|
|
1007
|
+
* Merge persisted pass-1 markers into the in-memory set (recovery reload).
|
|
1008
|
+
* Live markers win on conflict: they may already be partially consumed,
|
|
1009
|
+
* and replacing them with the older persisted total would double-count.
|
|
1010
|
+
* Entries are sanitized and the clock is freshened (see below).
|
|
1011
|
+
* @param {unknown} payload - `{ [orderId]: { filledInt, at } }`
|
|
1012
|
+
* @returns {void}
|
|
1013
|
+
*/
|
|
1014
|
+
restoreChainSizeAppliedFills(payload) {
|
|
1015
|
+
if (!(this._chainSizeAppliedFills instanceof Map))
|
|
1016
|
+
this._chainSizeAppliedFills = new Map();
|
|
1017
|
+
if (!payload || typeof payload !== 'object')
|
|
1018
|
+
return;
|
|
1019
|
+
const now = Date.now();
|
|
1020
|
+
// A persisted marker exists only because pass 1 applied the size but
|
|
1021
|
+
// the fill history had NOT been processed when the snapshot was
|
|
1022
|
+
// written. Freshen its clock on restore: the 5-minute TTL then covers
|
|
1023
|
+
// the post-restart replay window instead of rejecting evidence simply
|
|
1024
|
+
// because the process was down longer than the live TTL.
|
|
1025
|
+
for (const [id, raw] of Object.entries(payload)) {
|
|
1026
|
+
const entry = raw;
|
|
1027
|
+
const filledInt = Math.round(Number(entry?.filledInt));
|
|
1028
|
+
const at = Number(entry?.at);
|
|
1029
|
+
if (!id || !Number.isFinite(filledInt) || filledInt <= 0 || !Number.isFinite(at))
|
|
1030
|
+
continue;
|
|
1031
|
+
if (this._chainSizeAppliedFills.has(id))
|
|
1032
|
+
continue;
|
|
1033
|
+
this._chainSizeAppliedFills.set(id, { filledInt, at: now });
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
/** Snapshot the live pass-1 markers for persistence (empty when none). */
|
|
1037
|
+
snapshotChainSizeAppliedFills() {
|
|
1038
|
+
const out = {};
|
|
1039
|
+
if (!(this._chainSizeAppliedFills instanceof Map))
|
|
1040
|
+
return out;
|
|
1041
|
+
const now = Date.now();
|
|
1042
|
+
const ttl = timingWindowMs('CHAIN_SIZE_APPLIED_MARKER_TTL_MS', 5 * 60 * 1000);
|
|
1043
|
+
for (const [id, entry] of this._chainSizeAppliedFills) {
|
|
1044
|
+
const filledInt = Math.round(Number(entry?.filledInt));
|
|
1045
|
+
const at = Number(entry?.at);
|
|
1046
|
+
if (!id || !Number.isFinite(filledInt) || filledInt <= 0 || !Number.isFinite(at))
|
|
1047
|
+
continue;
|
|
1048
|
+
if (now - at > ttl)
|
|
1049
|
+
continue;
|
|
1050
|
+
out[id] = { filledInt, at };
|
|
1051
|
+
}
|
|
1052
|
+
return out;
|
|
1053
|
+
}
|
|
1054
|
+
/**
|
|
1055
|
+
* Wake the tasks that deferred on isPlacementActive() when the region ends.
|
|
1056
|
+
* The broadcast-region-end listener reschedules a deferred rebalance and
|
|
1057
|
+
* drains queued fills; without this a fill deferred because a correction
|
|
1058
|
+
* drain/COW held placement would only retry on the next unrelated event.
|
|
1059
|
+
* Skipped while a broadcast region is still active (stopBroadcasting will
|
|
1060
|
+
* fire the same listeners when it ends).
|
|
1061
|
+
* @returns {void}
|
|
1062
|
+
*/
|
|
1063
|
+
_handlePlacementRegionEnd() {
|
|
1064
|
+
if (this._broadcastingFlag > 0)
|
|
1065
|
+
return;
|
|
1066
|
+
this._fireBroadcastRegionEnd();
|
|
1067
|
+
}
|
|
717
1068
|
/**
|
|
718
1069
|
* @param {boolean} value
|
|
719
1070
|
* @returns {void}
|
|
720
1071
|
*/
|
|
721
1072
|
setShuttingDown(value) {
|
|
722
1073
|
this._shuttingDown = value === true;
|
|
1074
|
+
if (this._shuttingDown) {
|
|
1075
|
+
// Never let a hung placement region block shutdown/exit, and force
|
|
1076
|
+
// every waiter to abandon its batch instead of broadcasting.
|
|
1077
|
+
this._placementLock.forceRelease();
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
/**
|
|
1081
|
+
* Auto-clears a stale placement region after
|
|
1082
|
+
* TIMING.PLACEMENT_STALE_CLEAR_MS so a hung chain call / missed release
|
|
1083
|
+
* cannot park fills and rebalances forever. The holder's lease becomes a
|
|
1084
|
+
* no-op on release (token guard), so a late finally cannot release a
|
|
1085
|
+
* successor.
|
|
1086
|
+
*
|
|
1087
|
+
* TWO GUARDS, ONE TEARDOWN:
|
|
1088
|
+
* - A bot-level batch/recovery still in flight and YOUNGER than the bound is
|
|
1089
|
+
* evidence of progress, so the region is not torn down (a long create-group
|
|
1090
|
+
* reconcile must survive). Only a batch older than the same bound is
|
|
1091
|
+
* treated as part of the hung region.
|
|
1092
|
+
* - When the teardown does happen it also clears a broadcast refcount that
|
|
1093
|
+
* is itself past its bound, atomically. A refcount raised by a placement
|
|
1094
|
+
* holder belongs to the same region (the COW body and the reconcile call
|
|
1095
|
+
* startBroadcasting() INSIDE it), so clearing both in one place stops the
|
|
1096
|
+
* maintenance tick from tearing down placement first and the refcount
|
|
1097
|
+
* second (two authorities, one hung region). A refcount raised OUTSIDE any
|
|
1098
|
+
* region (the spread-correction pre-broadcast hold in `order/grid.ts`) is
|
|
1099
|
+
* not this region's and is left to `_clearStaleBroadcastFlag`, which still
|
|
1100
|
+
* refuses to act on its own while a placement lease is held.
|
|
1101
|
+
* @param {number} [maxAgeMs] - Override for tests
|
|
1102
|
+
* @param {Object} [options] - `batchInFlightMs`: age of the LAST raised
|
|
1103
|
+
* bot-level pipeline counter (0/undefined when none).
|
|
1104
|
+
* @returns {boolean} true when a stale region was force-released
|
|
1105
|
+
*/
|
|
1106
|
+
_clearStalePlacementRegion(maxAgeMs = TIMING.PLACEMENT_STALE_CLEAR_MS, options = {}) {
|
|
1107
|
+
if (!this.isPlacementActive())
|
|
1108
|
+
return false;
|
|
1109
|
+
const heldMs = this.placementHeldForMs();
|
|
1110
|
+
const limit = Number(maxAgeMs) > 0 ? Number(maxAgeMs) : TIMING.PLACEMENT_STALE_CLEAR_MS;
|
|
1111
|
+
if (heldMs <= limit)
|
|
1112
|
+
return false;
|
|
1113
|
+
const batchInFlightMs = Number(options?.batchInFlightMs);
|
|
1114
|
+
if (Number.isFinite(batchInFlightMs) && batchInFlightMs > 0 && batchInFlightMs <= limit) {
|
|
1115
|
+
this.logger?.log?.(`[PLACEMENT] Watchdog skipped: region held by ${this.placementKind() || 'unknown'} for ${Math.round(heldMs / 1000)}s, ` +
|
|
1116
|
+
`but the pipeline is still making progress (in flight ${Math.round(batchInFlightMs / 1000)}s)`, 'debug');
|
|
1117
|
+
return false;
|
|
1118
|
+
}
|
|
1119
|
+
const kind = this.placementKind() || 'unknown';
|
|
1120
|
+
this.logger?.log?.(`[PLACEMENT] Force-releasing stale placement region held by ${kind} for ${Math.round(heldMs / 1000)}s ` +
|
|
1121
|
+
`(watchdog ${Math.round(limit / 1000)}s); fills/rebalances must not stay parked`, 'error');
|
|
1122
|
+
this._placementLock.forceRelease();
|
|
1123
|
+
// Same hung region: drop its broadcast refcount too (see JSDoc).
|
|
1124
|
+
if (this._broadcastingFlag > 0 && this._broadcastingStartedAt > 0) {
|
|
1125
|
+
const broadcastAgeMs = Date.now() - this._broadcastingStartedAt;
|
|
1126
|
+
if (broadcastAgeMs > TIMING.BROADCAST_STALE_CLEAR_MS) {
|
|
1127
|
+
this.logger?.log?.(`[PLACEMENT] Also clearing the stale broadcast region (${Math.round(broadcastAgeMs / 1000)}s) it held; ` +
|
|
1128
|
+
`one teardown for one hung region`, 'warn');
|
|
1129
|
+
this._broadcastingFlag = 0;
|
|
1130
|
+
this._broadcastingStartedAt = 0;
|
|
1131
|
+
this._fireBroadcastRegionEnd();
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
return true;
|
|
723
1135
|
}
|
|
724
1136
|
/**
|
|
725
1137
|
* Auto-clears a stale broadcast flag after 120s so a hung
|
|
@@ -733,6 +1145,14 @@ class OrderManager {
|
|
|
733
1145
|
if (this._broadcastingFlag > 0 && this._broadcastingStartedAt > 0) {
|
|
734
1146
|
const elapsed = Date.now() - this._broadcastingStartedAt;
|
|
735
1147
|
if (elapsed > TIMING.BROADCAST_STALE_CLEAR_MS) {
|
|
1148
|
+
// A legitimately long reconcile/COW can hold placement well past
|
|
1149
|
+
// the watchdog while its refcount is still valid. Never cut the
|
|
1150
|
+
// region out from under a live placement holder: the refcount
|
|
1151
|
+
// is the leak signal only when nobody owns the region.
|
|
1152
|
+
if (this.isPlacementActive()) {
|
|
1153
|
+
this.logger?.log?.(`[BROADCAST] Stale-flag watchdog skipped: placement region still held by ${this.placementKind() || 'unknown'} for ${Math.round(this.placementHeldForMs() / 1000)}s`, 'debug');
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
736
1156
|
this.logger?.log?.(`[BROADCAST] Auto-clearing stale broadcast flag after ${Math.round(TIMING.BROADCAST_STALE_CLEAR_MS / 1000)}s`, 'warn');
|
|
737
1157
|
// Hard-reset to 0 (not decrement) — this is a safety valve for
|
|
738
1158
|
// a hung broadcast where stopBroadcasting() was never called.
|
|
@@ -1290,6 +1710,42 @@ class OrderManager {
|
|
|
1290
1710
|
}
|
|
1291
1711
|
}
|
|
1292
1712
|
}
|
|
1713
|
+
/**
|
|
1714
|
+
* The single answer to "does this order exist on chain, and how sure are we".
|
|
1715
|
+
*
|
|
1716
|
+
* Every consumer that used to infer this from one of the legacy maps must
|
|
1717
|
+
* read it here instead — that is the whole point of the lifecycle registry:
|
|
1718
|
+
* four sites consulting four different structures is how the engine ended up
|
|
1719
|
+
* with four different answers for the same order.
|
|
1720
|
+
*
|
|
1721
|
+
* A `createUncertain` slot flag (persisted across restarts) is folded into
|
|
1722
|
+
* `phase === 'unknown'` here, so no caller has to remember to check it.
|
|
1723
|
+
*
|
|
1724
|
+
* @param identity - Slot id, chain order id, or pending-broadcast fingerprint
|
|
1725
|
+
* @returns {LifecycleView} Unified read surface (never null)
|
|
1726
|
+
*/
|
|
1727
|
+
getLifecycle(identity) {
|
|
1728
|
+
return this._lifecycle.view(identity);
|
|
1729
|
+
}
|
|
1730
|
+
/**
|
|
1731
|
+
* Lifecycle view for a slot, folding in the slot's own persisted flags.
|
|
1732
|
+
* Prefer this over `getLifecycle(slotId)` wherever a slot object is at hand.
|
|
1733
|
+
* @param slot - Grid slot (ManagedOrder) or null
|
|
1734
|
+
* @returns {LifecycleView}
|
|
1735
|
+
*/
|
|
1736
|
+
getSlotLifecycle(slot) {
|
|
1737
|
+
if (!slot)
|
|
1738
|
+
return this._lifecycle.view(null);
|
|
1739
|
+
const identity = slot.orderId || slot.id || null;
|
|
1740
|
+
if (isSlotCreateUncertain(slot)) {
|
|
1741
|
+
// The persisted marker outranks an older registry phase: a CREATE
|
|
1742
|
+
// whose receipt was lost must never read as "nothing there".
|
|
1743
|
+
// Read-only: `withPhase` reports `unknown` without recording it, so a
|
|
1744
|
+
// read cannot keep the unresolved bound alive by refreshing its clock.
|
|
1745
|
+
return this._lifecycle.withPhase(identity, 'unknown', 'broadcast-receipt');
|
|
1746
|
+
}
|
|
1747
|
+
return this._lifecycle.view(identity);
|
|
1748
|
+
}
|
|
1293
1749
|
_normalizeOrderUpdateOptions(options = {}) {
|
|
1294
1750
|
if (options === null || typeof options !== 'object' || Array.isArray(options)) {
|
|
1295
1751
|
throw new TypeError('Order update options must be an object');
|
|
@@ -1363,41 +1819,11 @@ class OrderManager {
|
|
|
1363
1819
|
}
|
|
1364
1820
|
const updatedOrder = deepFreeze({ ...nextOrder });
|
|
1365
1821
|
const id = order.id;
|
|
1366
|
-
//
|
|
1367
|
-
//
|
|
1368
|
-
//
|
|
1369
|
-
//
|
|
1370
|
-
|
|
1371
|
-
// younger than SYNC_LOCK_TIMEOUT_MS.
|
|
1372
|
-
if (updatedOrder.orderId && (!oldOrder?.orderId || oldOrder.orderId !== updatedOrder.orderId)) {
|
|
1373
|
-
// Prune stamps older than the guard window — they can never be
|
|
1374
|
-
// consulted again (the consumer defers only while the stamp is
|
|
1375
|
-
// younger than SYNC_LOCK_TIMEOUT_MS), so keeping them would leak one
|
|
1376
|
-
// entry per orderId on a perpetual-turnover bot.
|
|
1377
|
-
for (const [orderId, assignedAt] of this._orderIdAssignedAt) {
|
|
1378
|
-
if (Date.now() - assignedAt >= TIMING.SYNC_LOCK_TIMEOUT_MS) {
|
|
1379
|
-
this._orderIdAssignedAt.delete(orderId);
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
this._orderIdAssignedAt.set(updatedOrder.orderId, Date.now());
|
|
1383
|
-
}
|
|
1384
|
-
// Fresh-placement timestamp for the surplus-cancel grace window: a
|
|
1385
|
-
// slot gaining an orderId it did not have is a new placement (create,
|
|
1386
|
-
// adopt, rotation target), and surplus sweeps must not cancel it
|
|
1387
|
-
// before it has had time to prove itself. Same-id updates do not
|
|
1388
|
-
// refresh the stamp, so grace cannot extend forever. Snapshot/grid
|
|
1389
|
-
// load paths are excluded: reloaded orders are old by definition, and
|
|
1390
|
-
// stamping them would neuter surplus sweeps for 15 minutes after every
|
|
1391
|
-
// restart.
|
|
1392
|
-
try {
|
|
1393
|
-
const prevId = oldOrder?.orderId != null ? String(oldOrder.orderId) : '';
|
|
1394
|
-
const nextId = updatedOrder.orderId != null ? String(updatedOrder.orderId) : '';
|
|
1395
|
-
const isLoadContext = context === 'grid-load' || context === 'grid-init';
|
|
1396
|
-
if (nextId && nextId !== prevId && !isLoadContext) {
|
|
1397
|
-
recordOrderPlacement(this, nextId);
|
|
1398
|
-
}
|
|
1399
|
-
}
|
|
1400
|
-
catch { /* bookkeeping must never break order updates */ }
|
|
1822
|
+
// Project the mutation into the lifecycle registry. This is the ONLY
|
|
1823
|
+
// writer of the chain-existence facts (phase, assignedAt, placedAt): the
|
|
1824
|
+
// stamps used to be written a second time right here, which meant two
|
|
1825
|
+
// copies of every clock and an O(records) prune loop per order update.
|
|
1826
|
+
this._syncLifecycleFromMutation(oldOrder, updatedOrder, context);
|
|
1401
1827
|
const newMap = cloneMap(this.orders);
|
|
1402
1828
|
newMap.set(id, updatedOrder);
|
|
1403
1829
|
this.orders = Object.freeze(newMap);
|
|
@@ -1409,6 +1835,99 @@ class OrderManager {
|
|
|
1409
1835
|
this._markGridDirty();
|
|
1410
1836
|
return true;
|
|
1411
1837
|
}
|
|
1838
|
+
/**
|
|
1839
|
+
* Project one order mutation into the lifecycle registry.
|
|
1840
|
+
*
|
|
1841
|
+
* This is the single write funnel for chain existence: every mutation goes
|
|
1842
|
+
* through `_applyOrderUpdate`, so no call site has to remember which of the
|
|
1843
|
+
* seven legacy structures to touch. A slot is the record's identity, so a
|
|
1844
|
+
* chain-id change (rotation / re-place) updates the same record instead of
|
|
1845
|
+
* starting a parallel one.
|
|
1846
|
+
*
|
|
1847
|
+
* @param previous - Slot before the mutation (absent on create)
|
|
1848
|
+
* @param next - Slot after the mutation
|
|
1849
|
+
* @param context - Mutation context label
|
|
1850
|
+
* @returns {void}
|
|
1851
|
+
*/
|
|
1852
|
+
_syncLifecycleFromMutation(previous, next, context) {
|
|
1853
|
+
try {
|
|
1854
|
+
const slotId = String(next.id);
|
|
1855
|
+
const nextId = next.orderId != null && String(next.orderId).length > 0 ? String(next.orderId) : '';
|
|
1856
|
+
const prevId = previous?.orderId != null && String(previous.orderId).length > 0 ? String(previous.orderId) : '';
|
|
1857
|
+
// Persisted "the CREATE result was lost" marker outranks everything:
|
|
1858
|
+
// the order may be live, so the phase must never read as absent.
|
|
1859
|
+
if (isSlotCreateUncertain(next)) {
|
|
1860
|
+
this._lifecycle.markUnknown(nextId || null, { slotId, evidence: 'broadcast-receipt' });
|
|
1861
|
+
return;
|
|
1862
|
+
}
|
|
1863
|
+
if (!nextId) {
|
|
1864
|
+
// The slot holds no chain id. Marking it `intent` would claim
|
|
1865
|
+
// "nothing is out there", which we may only say when nothing
|
|
1866
|
+
// unresolved is on record — unless this very mutation is the
|
|
1867
|
+
// explicit clear of the create-uncertain marker (the load-time
|
|
1868
|
+
// sanitizer does exactly that after neutralising the slot).
|
|
1869
|
+
const clearedUncertainMarker = isSlotCreateUncertain(previous) && !isSlotCreateUncertain(next);
|
|
1870
|
+
if (clearedUncertainMarker) {
|
|
1871
|
+
this._lifecycle.markIntent({ slotId });
|
|
1872
|
+
}
|
|
1873
|
+
else {
|
|
1874
|
+
this._lifecycle.markIntentIfResolved(slotId);
|
|
1875
|
+
}
|
|
1876
|
+
return;
|
|
1877
|
+
}
|
|
1878
|
+
if (nextId !== prevId) {
|
|
1879
|
+
// A freshly assigned chain id (create/adopt broadcast in flight, or
|
|
1880
|
+
// not yet visible to a lagging read) must not read as "absent from
|
|
1881
|
+
// chain": absence decisions defer while it is younger than
|
|
1882
|
+
// SYNC_LOCK_TIMEOUT_MS (isRecentlyAssigned).
|
|
1883
|
+
this._lifecycle.markAssigned(slotId, nextId);
|
|
1884
|
+
// Fresh-placement timestamp for the surplus-cancel grace window: a
|
|
1885
|
+
// slot gaining an orderId it did not have is a new placement
|
|
1886
|
+
// (create, adopt, rotation target), and surplus sweeps must not
|
|
1887
|
+
// cancel it before it has had time to prove itself. Same-id updates
|
|
1888
|
+
// do not refresh the stamp, so grace cannot extend forever.
|
|
1889
|
+
// Snapshot/grid load paths are excluded: reloaded orders are old by
|
|
1890
|
+
// definition, and stamping them would neuter surplus sweeps for 15
|
|
1891
|
+
// minutes after every restart.
|
|
1892
|
+
const isLoadContext = context === 'grid-load' || context === 'grid-init';
|
|
1893
|
+
if (!isLoadContext)
|
|
1894
|
+
this._lifecycle.markPlaced(nextId, undefined, slotId);
|
|
1895
|
+
}
|
|
1896
|
+
// This funnel only knows the LOCAL slot state, not where that state
|
|
1897
|
+
// came from, so the evidence is 'local': it must not advance
|
|
1898
|
+
// `observedAt` ("when did the CHAIN last tell us"). The chain-backed
|
|
1899
|
+
// sites (sync pass 1 match, pass 2 adoption) call markLanded with
|
|
1900
|
+
// 'chain-snapshot' / 'adoption' themselves.
|
|
1901
|
+
if (next.state === ORDER_STATES.PARTIAL) {
|
|
1902
|
+
this._lifecycle.observe(nextId, { slotId, phase: 'filling', evidence: 'local' });
|
|
1903
|
+
}
|
|
1904
|
+
else if (next.state === ORDER_STATES.ACTIVE) {
|
|
1905
|
+
this._lifecycle.observe(nextId, { slotId, phase: 'landed', evidence: 'local' });
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
catch { /* lifecycle bookkeeping must never break an order update */ }
|
|
1909
|
+
}
|
|
1910
|
+
/**
|
|
1911
|
+
* Abandon an unresolved pending CREATE (TTL sweep / give-up).
|
|
1912
|
+
*
|
|
1913
|
+
* Distinct from `_pendingBroadcasts.delete()`: dropping an entry because the
|
|
1914
|
+
* create RESOLVED leaves the slot free, while dropping it because we ran out
|
|
1915
|
+
* of patience means the order may still be live. Collapsing both into one
|
|
1916
|
+
* delete is how a lost receipt turned into a clean hole that got re-CREATEd.
|
|
1917
|
+
*
|
|
1918
|
+
* @param fingerprint - Pending-broadcast fingerprint being abandoned
|
|
1919
|
+
* @param entry - The abandoned entry (carries the slot id)
|
|
1920
|
+
* @returns {void}
|
|
1921
|
+
*/
|
|
1922
|
+
_expirePendingBroadcast(fingerprint, entry) {
|
|
1923
|
+
const slotId = entry?.slotId != null ? String(entry.slotId) : null;
|
|
1924
|
+
try {
|
|
1925
|
+
this._lifecycle.deletePending(fingerprint);
|
|
1926
|
+
if (slotId)
|
|
1927
|
+
this._lifecycle.markUnknown(null, { slotId, evidence: 'local' });
|
|
1928
|
+
}
|
|
1929
|
+
catch { /* bookkeeping only */ }
|
|
1930
|
+
}
|
|
1412
1931
|
_peekWorkingGrid() {
|
|
1413
1932
|
const stack = this._currentWorkingGridStack;
|
|
1414
1933
|
return stack.length > 0 ? stack[stack.length - 1] : null;
|
|
@@ -1419,7 +1938,16 @@ class OrderManager {
|
|
|
1419
1938
|
return;
|
|
1420
1939
|
}
|
|
1421
1940
|
try {
|
|
1422
|
-
|
|
1941
|
+
// A mutation during a structural rebuild is ALWAYS structural, even
|
|
1942
|
+
// when its own context (`sync-pass1-partial`, `sync-pass2-orphan`,
|
|
1943
|
+
// `fill-place`) looks like a fill. The rebuild's fresh master has no
|
|
1944
|
+
// live chain ids, so a create-only re-plan at these levels duplicates
|
|
1945
|
+
// the reconcile's update-first placement. Stamping the reason here
|
|
1946
|
+
// makes the classifier independent of the individual sync context.
|
|
1947
|
+
const reason = this.isStructuralRebuildInProgress()
|
|
1948
|
+
? `master mutation during structural rebuild (${context})`
|
|
1949
|
+
: `master mutation during ${(this._rebalanceState || '').toLowerCase()} (${context})`;
|
|
1950
|
+
wg.markStale(reason);
|
|
1423
1951
|
wg.syncFromMaster(this.orders, orderId, this._gridVersion);
|
|
1424
1952
|
}
|
|
1425
1953
|
catch (syncErr) {
|
|
@@ -1535,6 +2063,13 @@ class OrderManager {
|
|
|
1535
2063
|
// instead of waiting in-lock; processFillsOnly already recorded
|
|
1536
2064
|
// the boundary crawls, so a later derivation applies them. The
|
|
1537
2065
|
// region-end hook schedules the deferred rebalance.
|
|
2066
|
+
//
|
|
2067
|
+
// RE-ANCHOR BUDGET SNAPSHOT: the absolute fill anchor is a per-batch
|
|
2068
|
+
// budget spent at PLAN time (strategy.calculateTargetGrid). Carry the
|
|
2069
|
+
// pre-plan value so a stale-plan re-plan can hand back exactly what
|
|
2070
|
+
// the abandoned plan spent (a static batch base would wrongly refund
|
|
2071
|
+
// an anchor an earlier chunk legitimately committed).
|
|
2072
|
+
const reanchorBudgetBefore = Number(this._boundaryReanchorBudget);
|
|
1538
2073
|
const rebalanceResult = await this.performSafeRebalance(orders, excl, {
|
|
1539
2074
|
...(options || {}),
|
|
1540
2075
|
deferIfBroadcasting: true
|
|
@@ -1544,6 +2079,7 @@ class OrderManager {
|
|
|
1544
2079
|
// stale between planning and broadcast.
|
|
1545
2080
|
rebalanceResult.fills = orders;
|
|
1546
2081
|
rebalanceResult.excludeIds = excl || new Set();
|
|
2082
|
+
rebalanceResult._reanchorBudgetBeforePlan = reanchorBudgetBefore;
|
|
1547
2083
|
return rebalanceResult;
|
|
1548
2084
|
}
|
|
1549
2085
|
const workingGrid = new WorkingGrid(this.orders, { baseVersion: this._gridVersion });
|
|
@@ -1824,8 +2360,23 @@ class OrderManager {
|
|
|
1824
2360
|
const boundary = newIdx == null ? this.boundaryIdx : newIdx;
|
|
1825
2361
|
if (boundary == null)
|
|
1826
2362
|
return;
|
|
1827
|
-
|
|
2363
|
+
// Geometry: resolveGapSlots is the single source of truth
|
|
2364
|
+
// (_gapSlots ?? config-derived). Reading the raw field with a `0`
|
|
2365
|
+
// fallback printed `gapSlots=0 sellStart=boundary+1` for every
|
|
2366
|
+
// restore that happens before the loader publishes _gapSlots —
|
|
2367
|
+
// indistinguishable in the log from a genuinely collapsed spread
|
|
2368
|
+
// band (field report: "bot lost its fixed spread, only one slot is
|
|
2369
|
+
// used"). A config-derived gap is always a valid geometry; an
|
|
2370
|
+
// absent one is unknown, never zero.
|
|
2371
|
+
const gapSlots = resolveGapSlots(this);
|
|
1828
2372
|
const sellStart = boundary + 1 + gapSlots;
|
|
2373
|
+
// The spread metric only exists once a grid has been loaded or
|
|
2374
|
+
// created (both counters are 0 on a fresh manager, which is
|
|
2375
|
+
// indistinguishable from a genuinely empty band). Printing 0
|
|
2376
|
+
// before the measurement claims an empty band we have not measured.
|
|
2377
|
+
const spreadCount = this._spreadCountKnown
|
|
2378
|
+
? (this.currentSpreadCount ?? this.initialSpreadCount)
|
|
2379
|
+
: '?';
|
|
1829
2380
|
const countActive = (type) => {
|
|
1830
2381
|
let n = 0;
|
|
1831
2382
|
for (const o of this.orders.values()) {
|
|
@@ -1835,7 +2386,7 @@ class OrderManager {
|
|
|
1835
2386
|
return n;
|
|
1836
2387
|
};
|
|
1837
2388
|
this.logger?.log?.(`[BOUNDARY] ${source} boundary=${boundary} sellStart=${sellStart} gapSlots=${gapSlots} ` +
|
|
1838
|
-
`spreadCount=${
|
|
2389
|
+
`spreadCount=${spreadCount} ` +
|
|
1839
2390
|
`activeBuy=${countActive(ORDER_TYPES.BUY)} activeSell=${countActive(ORDER_TYPES.SELL)}`, 'debug');
|
|
1840
2391
|
}
|
|
1841
2392
|
catch {
|
|
@@ -2341,6 +2892,38 @@ class OrderManager {
|
|
|
2341
2892
|
// _fillProcessingLock (the lock's acquisition timeout is 20s; the
|
|
2342
2893
|
// wait is 30s). A fill-driven rebalance defers instead, leaving
|
|
2343
2894
|
// the crawls recorded by processFillsOnly for a later derivation.
|
|
2895
|
+
//
|
|
2896
|
+
// STRUCTURAL REBUILD DEFER (2026-11-03 live duplicate-order incident): a
|
|
2897
|
+
// trigger-file rebuild rebuilds + reconciles the grid. A
|
|
2898
|
+
// fill-driven COW that plans during that window re-plans against
|
|
2899
|
+
// the freshly rebuilt (order-id-less) master and emits a
|
|
2900
|
+
// create-only plan, while startup reconcile re-prices the live
|
|
2901
|
+
// orders onto the same levels — both generations land as
|
|
2902
|
+
// duplicate-price orphans. The reconcile owns placement until
|
|
2903
|
+
// endStructuralRebuild(); fills stay queued and are drained by the
|
|
2904
|
+
// bootstrap consumer. This check must run before
|
|
2905
|
+
// deferIfBroadcasting because the fill path passes that flag.
|
|
2906
|
+
if (this.isStructuralRebuildInProgress()) {
|
|
2907
|
+
const rebuildAgeMs = this._structuralRebuildStartedAt > 0
|
|
2908
|
+
? Date.now() - this._structuralRebuildStartedAt
|
|
2909
|
+
: 0;
|
|
2910
|
+
const deferBoundMs = timingWindowMs('STRUCTURAL_REBUILD_DEFER_MAX_MS', 3 * 60 * 1000);
|
|
2911
|
+
const overBound = rebuildAgeMs > deferBoundMs;
|
|
2912
|
+
const lastWarnAt = Number(this._structuralRebuildDeferWarnAt) || 0;
|
|
2913
|
+
const shouldWarn = overBound && Date.now() - lastWarnAt > 60000;
|
|
2914
|
+
if (shouldWarn) {
|
|
2915
|
+
this._structuralRebuildDeferWarnAt = Date.now();
|
|
2916
|
+
}
|
|
2917
|
+
this.logger.log(`[SAFE-REBALANCE] Deferred: structural rebuild in progress (${this.structuralRebuildReason() || 'rebuild'}, ` +
|
|
2918
|
+
`${Math.round(rebuildAgeMs / 1000)}s); startup reconcile owns placement`, shouldWarn ? 'warn' : (overBound ? 'debug' : 'info'));
|
|
2919
|
+
this._deferredRebalanceAt = Date.now();
|
|
2920
|
+
return { ...buildAbortedResult('structural-rebuild-deferred'), deferred: true };
|
|
2921
|
+
}
|
|
2922
|
+
if (this.isPlacementActive()) {
|
|
2923
|
+
this.logger.log(`[SAFE-REBALANCE] Deferred: placement region held by ${this.placementKind() || 'another path'}; rebalance retries after it ends`, 'info');
|
|
2924
|
+
this._deferredRebalanceAt = Date.now();
|
|
2925
|
+
return { ...buildAbortedResult('placement-active-deferred'), deferred: true };
|
|
2926
|
+
}
|
|
2344
2927
|
if (options?.deferIfBroadcasting && this.isBroadcastingActive()) {
|
|
2345
2928
|
this.logger.log('[SAFE-REBALANCE] Deferred: broadcast/placement activity active; rebalance retries after the region ends', 'info');
|
|
2346
2929
|
this._deferredRebalanceAt = Date.now();
|
|
@@ -2413,6 +2996,12 @@ class OrderManager {
|
|
|
2413
2996
|
// delay the cancel-only evacuation queue. Queued-once per slot; the
|
|
2414
2997
|
// corrections runner (correctAllPriceMismatches) executes the cancel
|
|
2415
2998
|
// and settles the slot back to a spread placeholder.
|
|
2999
|
+
//
|
|
3000
|
+
// The COW pipeline holds a queued cancel back from its pre-batch drain
|
|
3001
|
+
// when the same batch ROTATES that chain order (the rotation IS the
|
|
3002
|
+
// evacuation) — see updateOrdersOnChainBatchCOWBody. Queueing both is
|
|
3003
|
+
// therefore safe: the rotation runs, and the held entry is dropped as
|
|
3004
|
+
// stale on the next drain because the slot no longer owns the order.
|
|
2416
3005
|
try {
|
|
2417
3006
|
this._processGapEvacuationTeeth(result?.evacReady);
|
|
2418
3007
|
}
|
|
@@ -2537,6 +3126,21 @@ class OrderManager {
|
|
|
2537
3126
|
deepFreeze(order);
|
|
2538
3127
|
}
|
|
2539
3128
|
}
|
|
3129
|
+
// LIFE-CYCLE PROJECTION OF THE WHOLE-MAP SWAP.
|
|
3130
|
+
// The COW commit replaces `this.orders` wholesale instead of
|
|
3131
|
+
// calling _applyOrderUpdate per slot, so it used to bypass the
|
|
3132
|
+
// single write funnel entirely: an order the batch just PLACED got
|
|
3133
|
+
// no `assignedAt` and — worse — no `placedAt`, leaving the 15-minute
|
|
3134
|
+
// surplus-cancel grace inert for the primary placement path (a
|
|
3135
|
+
// competing sweep could cancel a seconds-old order). Projecting each
|
|
3136
|
+
// slot through the same funnel closes that hole; the funnel only
|
|
3137
|
+
// records bookkeeping, it does not change what is committed.
|
|
3138
|
+
// Runs BEFORE the swap so each projection sees the previous slot.
|
|
3139
|
+
for (const [slotId, nextOrder] of finalMap.entries()) {
|
|
3140
|
+
if (!nextOrder)
|
|
3141
|
+
continue;
|
|
3142
|
+
this._syncLifecycleFromMutation(this.orders.get(slotId), nextOrder, 'cow-commit');
|
|
3143
|
+
}
|
|
2540
3144
|
this.orders = Object.freeze(finalMap);
|
|
2541
3145
|
this._setBoundary(commitBoundary);
|
|
2542
3146
|
// Pending-crawl bookkeeping: an accepted non-null commit
|
|
@@ -2566,16 +3170,15 @@ class OrderManager {
|
|
|
2566
3170
|
this._gridVersion++;
|
|
2567
3171
|
committed = true;
|
|
2568
3172
|
// Track orderIds from successful COW commits for recovery sync protection.
|
|
2569
|
-
//
|
|
2570
|
-
//
|
|
2571
|
-
//
|
|
3173
|
+
// ONE publish: the snapshot setter stamps every id AND moves the
|
|
3174
|
+
// built-at window, so a second `_committedOrderIdsBuiltAt` write
|
|
3175
|
+
// would only re-stamp the same ids with a later clock.
|
|
2572
3176
|
const newCommittedIds = new Set();
|
|
2573
3177
|
for (const [, order] of finalMap.entries()) {
|
|
2574
3178
|
if (order.orderId)
|
|
2575
3179
|
newCommittedIds.add(order.orderId);
|
|
2576
3180
|
}
|
|
2577
3181
|
this._committedOrderIds = newCommittedIds;
|
|
2578
|
-
this._committedOrderIdsBuiltAt = Date.now();
|
|
2579
3182
|
// Index caches invalidated by _gridVersion bump above — lazy getters recompute from this.orders
|
|
2580
3183
|
});
|
|
2581
3184
|
if (!committed) {
|