dexbot 1.7.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +3 -3
- package/analysis/README.md +33 -13
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/account_resolver.js +5 -6
- package/dist/analysis/analyze_liquidity.js +15 -4
- package/dist/analysis/bot_key_utils.d.ts +44 -1
- package/dist/analysis/bot_key_utils.js +76 -7
- package/dist/analysis/fills_cache.js +2 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -19
- package/dist/analysis/grid_correction_check.js +12 -30
- package/dist/analysis/liquidity/liquidity_chart_generator.js +190 -20
- package/dist/analysis/liquidity/liquidity_depth_model.d.ts +41 -16
- package/dist/analysis/liquidity/liquidity_depth_model.js +49 -21
- package/dist/analysis/liquidity/liquidity_view.d.ts +15 -0
- package/dist/analysis/liquidity/liquidity_view.js +11 -0
- package/dist/analysis/resolve_bot_accounts.js +7 -6
- package/dist/analysis/trade_profitability.js +2 -1
- package/dist/credential-daemon.js +13 -2
- package/dist/dexbot.js +4 -4
- package/dist/modules/account_bots.d.ts +11 -1
- package/dist/modules/account_orders.d.ts +12 -0
- package/dist/modules/account_orders.js +30 -0
- package/dist/modules/bitshares-native/transport.js +24 -6
- package/dist/modules/bitshares_client.d.ts +24 -6
- package/dist/modules/bitshares_client.js +44 -7
- package/dist/modules/bot_settings.d.ts +2 -0
- package/dist/modules/constants.d.ts +26 -3
- package/dist/modules/constants.js +63 -8
- package/dist/modules/dexbot_class.d.ts +1 -6
- package/dist/modules/dexbot_class.js +14 -0
- package/dist/modules/dexbot_cow_runtime.d.ts +97 -7
- package/dist/modules/dexbot_cow_runtime.js +620 -132
- package/dist/modules/dexbot_fill_runtime.js +124 -35
- package/dist/modules/dexbot_maintenance_runtime.js +226 -73
- package/dist/modules/dexbot_startup_runtime.js +32 -20
- package/dist/modules/dexbot_state_recovery.d.ts +33 -1
- package/dist/modules/dexbot_state_recovery.js +167 -5
- package/dist/modules/node_connect_policy.d.ts +8 -1
- package/dist/modules/node_connect_policy.js +18 -2
- package/dist/modules/node_flap_score.d.ts +87 -0
- package/dist/modules/node_flap_score.js +124 -0
- package/dist/modules/node_health_cache.d.ts +40 -3
- package/dist/modules/node_health_cache.js +66 -20
- package/dist/modules/node_manager.d.ts +101 -19
- package/dist/modules/node_manager.js +255 -65
- package/dist/modules/order/accounting.js +56 -7
- package/dist/modules/order/format.d.ts +24 -10
- package/dist/modules/order/format.js +30 -10
- package/dist/modules/order/grid.js +82 -11
- package/dist/modules/order/grid_reconcile.d.ts +5 -1
- package/dist/modules/order/grid_reconcile.js +333 -210
- package/dist/modules/order/grid_reconcile_internal.d.ts +38 -2
- package/dist/modules/order/grid_reconcile_internal.js +207 -87
- package/dist/modules/order/index.d.ts +4 -0
- package/dist/modules/order/lifecycle.d.ts +353 -0
- package/dist/modules/order/lifecycle.js +1109 -0
- package/dist/modules/order/manager.d.ts +271 -5
- package/dist/modules/order/manager.js +651 -48
- package/dist/modules/order/placement_lock.d.ts +102 -0
- package/dist/modules/order/placement_lock.js +222 -0
- package/dist/modules/order/strategy.js +13 -1
- package/dist/modules/order/sync_engine.d.ts +13 -1
- package/dist/modules/order/sync_engine.js +153 -53
- package/dist/modules/order/utils/math.d.ts +22 -1
- package/dist/modules/order/utils/math.js +54 -1
- package/dist/modules/order/utils/order.d.ts +75 -5
- package/dist/modules/order/utils/order.js +158 -9
- package/dist/modules/order/utils/system.d.ts +18 -0
- package/dist/modules/order/utils/system.js +84 -8
- package/dist/modules/order/utils/validate.d.ts +7 -1
- package/dist/modules/order/utils/validate.js +48 -2
- package/dist/modules/outcome.d.ts +54 -0
- package/dist/modules/outcome.js +51 -0
- package/dist/modules/settings_merge.d.ts +4 -0
- package/dist/modules/settings_merge.js +9 -0
- package/dist/modules/timing_policy.d.ts +82 -0
- package/dist/modules/timing_policy.js +244 -0
- package/dist/modules/types.d.ts +132 -0
- package/dist/scripts/analyze-credit.js +54 -17
- package/dist/scripts/analyze-orders.d.ts +4 -2
- package/dist/scripts/analyze-orders.js +113 -34
- package/dist/scripts/chart_targets.js +11 -15
- package/dist/scripts/test-credit-renewal.js +12 -5
- package/docs/COW_INVARIANTS.md +67 -8
- package/docs/DEXBOT_COMPARISON.md +6 -6
- package/docs/EVOLUTION.md +9 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_PRICE_INVARIANT.md +1 -1
- package/docs/GRID_RECONCILE.md +17 -12
- package/docs/LOGGING.md +25 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +175 -26
- package/docs/README.md +23 -2
- package/docs/WORKFLOW.md +9 -3
- package/docs/architecture.md +42 -5
- package/docs/developer_guide.md +25 -8
- package/modules/README.md +3 -0
- package/package.json +1 -1
- package/scripts/README.md +16 -0
- package/scripts/pdev.sh +14 -1
- package/scripts/pmain.sh +23 -8
- package/scripts/ptest.sh +7 -0
- package/tests/README.md +30 -1
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { createRequire } from 'node:module';
|
|
2
2
|
const require = createRequire(import.meta.url);
|
|
3
3
|
import { _countActiveOnGrid, _cancelChainOrder, _recoverStartupSyncFailure, _refreshStartupUpdatePlans, _executeStartupUpdateBatch, _executeStartupSequentialUpdateFallback, _executePlannedStartupCreates, _reconcileStartupSide, } from './grid_reconcile_internal.js';
|
|
4
|
-
import { ORDER_TYPES, ORDER_STATES
|
|
4
|
+
import { ORDER_TYPES, ORDER_STATES } from '../constants.js';
|
|
5
5
|
import { readOpenOrdersGuarded } from '../chain_orders.js';
|
|
6
|
-
import { getAssetFeesSafe, pricesOnSameLevel } from './utils/math.js';
|
|
7
|
-
import { isOrderPlaced, parseChainOrder, isOrderOnChain, findLiveOrderOwnerByChainId, chainOrderMatchesSlotWithTolerance, duplicateOrphanLogInfo, resolveEdgeCount, reviveVirtualSlotFromChain, } from './utils/order.js';
|
|
6
|
+
import { getAssetFeesSafe, pricesOnSameLevel, resolveGapSlots } from './utils/math.js';
|
|
7
|
+
import { isOrderPlaced, parseChainOrder, isOrderOnChain, findLiveOrderOwnerByChainId, chainOrderMatchesSlotWithTolerance, duplicateOrphanLogInfo, resolveEdgeCount, reviveVirtualSlotFromChain, getLivePendingBroadcasts, } from './utils/order.js';
|
|
8
8
|
import * as Format from './format.js';
|
|
9
9
|
import { getErrorMessage } from '../utils/errors.js';
|
|
10
|
+
import { readSlotLifecycle } from './lifecycle.js';
|
|
10
11
|
function _startupChainSnapshotSignature(orders, manager) {
|
|
11
12
|
const signature = new Map();
|
|
12
13
|
for (const raw of Array.isArray(orders) ? orders : []) {
|
|
@@ -32,7 +33,11 @@ function _startupCancelPlanStillCurrent(plan, manager, initialChainSignature, cu
|
|
|
32
33
|
if (currentChainSignature.get(chainOrderId) !== initialChainSignature.get(chainOrderId))
|
|
33
34
|
return false;
|
|
34
35
|
if (plan.gridOrderId) {
|
|
35
|
-
|
|
36
|
+
// Geometry identity is compared through the same resolver the PLAN used
|
|
37
|
+
// (resolveGapSlots), never the raw `_gapSlots` field: an unpublished field
|
|
38
|
+
// would compare `undefined` against the plan's derived band and skip every
|
|
39
|
+
// cancellation as "stale".
|
|
40
|
+
if (plan.boundaryIdx !== undefined && (plan.boundaryIdx !== manager?.boundaryIdx || plan.gapSlots !== resolveGapSlots(manager)))
|
|
36
41
|
return false;
|
|
37
42
|
const slot = manager?.orders instanceof Map ? manager.orders.get(plan.gridOrderId) : null;
|
|
38
43
|
if (!slot || !isOrderOnChain(slot) || slot.orderId !== plan.chainOrderId)
|
|
@@ -157,7 +162,7 @@ export async function decideStartupGridAction({ persistedGrid, chainOpenOrders,
|
|
|
157
162
|
* @param {Array<Object>} params.chainOpenOrders - On-chain open orders
|
|
158
163
|
* @returns {Promise<Object|null>} Reconciliation result, or null when reconciliation completes without a summary object
|
|
159
164
|
*/
|
|
160
|
-
export async function reconcileGridOrders({ manager, config, account, privateKey, chainOrders, chainOpenOrders, }) {
|
|
165
|
+
export async function reconcileGridOrders({ manager, config, account, privateKey, chainOrders, chainOpenOrders, options = {}, }) {
|
|
161
166
|
// Parameter validation
|
|
162
167
|
if (!manager || typeof manager.synchronizeWithChain !== 'function') {
|
|
163
168
|
throw new Error('reconcileGridOrders: manager must be provided with synchronizeWithChain method');
|
|
@@ -181,207 +186,294 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
|
|
|
181
186
|
}
|
|
182
187
|
const logger = manager && manager.logger;
|
|
183
188
|
const dryRun = !!(config && config.dryRun);
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
189
|
+
// A maintenance/targeted reconcile must NOT broadcast while a structural
|
|
190
|
+
// rebuild owns placement: Phase 1 mutates the master and Phase 2 places
|
|
191
|
+
// orders, so running it inside the rebuild re-creates the duplicate class
|
|
192
|
+
// the rebuild's own reconcile is designed to avoid. Rebuild callers pass
|
|
193
|
+
// allowDuringRebuild (they ARE the rebuild and run update-first).
|
|
194
|
+
const rebuildInProgress = manager.isStructuralRebuildInProgress?.() === true;
|
|
195
|
+
if (rebuildInProgress && options.allowDuringRebuild !== true) {
|
|
196
|
+
logger?.log?.('Reconcile deferred: structural grid rebuild in progress; the rebuild owns placement and update-first repair', 'info');
|
|
197
|
+
return null;
|
|
198
|
+
}
|
|
199
|
+
// PHASE 1: In-memory reconciliation under lock — planning runs with no
|
|
200
|
+
// blockchain I/O INSIDE the lock. All cancellations are collected into
|
|
201
|
+
// plannedCancels and executed in Phase 2. This keeps the lock hierarchy
|
|
202
|
+
// clean (no _gridLock held across sync calls).
|
|
203
|
+
//
|
|
204
|
+
// The exclusive placement region is acquired BEFORE the chain snapshot is
|
|
205
|
+
// used: the caller read the chain before this call, and waiting up to 120s
|
|
206
|
+
// for placement lets another path broadcast in between. Planning from that
|
|
207
|
+
// stale view is the duplicate-class race. After the acquire we re-read the
|
|
208
|
+
// chain (guarded: non-truncated, non-empty) so Phase 1/2 work from a
|
|
209
|
+
// snapshot no older than the region. Held across Phase 1 + Phase 2 and
|
|
210
|
+
// released after stopBroadcasting in the finally below.
|
|
211
|
+
// TOKEN-LEVEL HOLDER: this function acquires and releases its own placement
|
|
212
|
+
// lease because it must span Phase 1 (planning) and Phase 2 (broadcasting)
|
|
213
|
+
// — a `withPlacement` callback body would invert the phase structure. Every
|
|
214
|
+
// OTHER chain-mutating path goes through `manager.withPlacement()`; keep it
|
|
215
|
+
// that way (see placement_lock.ts).
|
|
216
|
+
let placementLease = null;
|
|
217
|
+
let regionStarted = false;
|
|
218
|
+
const releaseRegion = () => {
|
|
219
|
+
if (regionStarted)
|
|
220
|
+
manager.stopBroadcasting?.();
|
|
221
|
+
placementLease?.release();
|
|
222
|
+
placementLease = null;
|
|
223
|
+
};
|
|
224
|
+
let chainSnapshot = Array.isArray(chainOpenOrders) ? chainOpenOrders : [];
|
|
187
225
|
const activeCfg = (config && config.coreOrders) ? config.coreOrders : {};
|
|
188
226
|
let targetBuy = Math.max(0, Number.isFinite(Number(activeCfg.buy)) ? Number(activeCfg.buy) : 1)
|
|
189
227
|
+ resolveEdgeCount(config, 'buy');
|
|
190
228
|
let targetSell = Math.max(0, Number.isFinite(Number(activeCfg.sell)) ? Number(activeCfg.sell) : 1)
|
|
191
229
|
+ resolveEdgeCount(config, 'sell');
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
throw new Error('manager._applyOrderUpdate is required for startup reconciliation');
|
|
200
|
-
}
|
|
201
|
-
const applyUpdate = manager._applyOrderUpdate.bind(manager);
|
|
202
|
-
const chainIds = new Set((Array.isArray(chainOpenOrders) ? chainOpenOrders : []).map(o => o?.id).filter((id) => Boolean(id)));
|
|
203
|
-
const chainOrderById = new Map();
|
|
204
|
-
for (const raw of (Array.isArray(chainOpenOrders) ? chainOpenOrders : [])) {
|
|
205
|
-
if (raw?.id)
|
|
206
|
-
chainOrderById.set(String(raw.id), raw);
|
|
207
|
-
}
|
|
208
|
-
for (const order of manager.orders.values()) {
|
|
209
|
-
if (isOrderPlaced(order)) {
|
|
210
|
-
if (!order.orderId || !chainIds.has(order.orderId)) {
|
|
211
|
-
// Absence-decision guard: only virtualize as a phantom when
|
|
212
|
-
// the absence is trustworthy. An orderId assigned within the
|
|
213
|
-
// sync-lock window may be an in-flight create/adopt whose
|
|
214
|
-
// broadcast has not landed or is not yet visible to a
|
|
215
|
-
// lagging/truncated read. Virtualizing it here and
|
|
216
|
-
// re-creating would duplicate a real live order (the
|
|
217
|
-
// reconcile-timeout death-spiral root cause). Mirror the
|
|
218
|
-
// sync_engine committed-order guard; ghost orders (PARTIAL +
|
|
219
|
-
// size=0) still pass through so known fills get cleaned up.
|
|
220
|
-
const assignedAt = manager._orderIdAssignedAt?.get(order.orderId ?? '') || 0;
|
|
221
|
-
const isGhost = order.size <= 0 && order.state === ORDER_STATES.PARTIAL;
|
|
222
|
-
if (!isGhost && assignedAt > 0 && Date.now() - assignedAt < TIMING.SYNC_LOCK_TIMEOUT_MS) {
|
|
223
|
-
logger?.log?.(`Startup: Order ${order.id} (ID ${order.orderId}) absent from snapshot but freshly assigned ` +
|
|
224
|
-
`(${Date.now() - assignedAt}ms ago); deferring phantom cleanup (duplicate-order protection).`, 'warn');
|
|
225
|
-
continue;
|
|
226
|
-
}
|
|
227
|
-
logger?.log?.(`Startup: Found phantom order ${order.id} (ID ${order.orderId}) not on chain. Resetting to VIRTUAL.`, 'warn');
|
|
228
|
-
await applyUpdate({
|
|
229
|
-
...order,
|
|
230
|
-
state: ORDER_STATES.VIRTUAL,
|
|
231
|
-
orderId: "",
|
|
232
|
-
rawOnChain: null
|
|
233
|
-
}, 'startup-phantom', { skipAccounting: true });
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
else if (order.orderId && chainIds.has(String(order.orderId))) {
|
|
237
|
-
// DARK LEVEL: the slot is not placed but its chain order is
|
|
238
|
-
// still live (a phantom cleanup on a truncated read, or a
|
|
239
|
-
// refused-commit adoption). Left as-is it is invisible to
|
|
240
|
-
// window accounting (_countActiveOnGrid counts only
|
|
241
|
-
// ACTIVE/PARTIAL) while its order still counts in chainCount —
|
|
242
|
-
// so this side reads a shortfall it can never repair, and the
|
|
243
|
-
// planner sees a free level that already holds an order (the
|
|
244
|
-
// duplicate-price precondition). Revive in place: no broadcast,
|
|
245
|
-
// and only when the order really rests at this slot's level.
|
|
246
|
-
const rawChain = chainOrderById.get(String(order.orderId)) ?? null;
|
|
247
|
-
const parsedChain = rawChain ? parseChainOrder(rawChain, manager.assets) : null;
|
|
248
|
-
// The helper logs the revive itself (one warn line, same text
|
|
249
|
-
// on both call sites) — no second log here.
|
|
250
|
-
await reviveVirtualSlotFromChain(manager, order, parsedChain, rawChain, 'startup-revive', { skipAccounting: true });
|
|
230
|
+
let phase1Result;
|
|
231
|
+
try {
|
|
232
|
+
if (typeof manager.acquirePlacement === 'function') {
|
|
233
|
+
placementLease = await manager.acquirePlacement('startup-reconcile', { timeoutMs: 120000 });
|
|
234
|
+
if (!placementLease) {
|
|
235
|
+
logger?.log?.('Startup reconcile: exclusive placement region unavailable (timeout/shutdown); deferring to the next sync cycle.', 'warn');
|
|
236
|
+
return null;
|
|
251
237
|
}
|
|
252
238
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
239
|
+
try {
|
|
240
|
+
const refreshed = await readOpenOrdersGuarded(chainOrders, account, {
|
|
241
|
+
log: (msg, level) => logger?.log?.(msg, level),
|
|
242
|
+
label: 'STARTUP-RECONCILE',
|
|
243
|
+
detail: 'post-placement refresh',
|
|
244
|
+
deferEmpty: true,
|
|
245
|
+
});
|
|
246
|
+
if (Array.isArray(refreshed) && refreshed.length > 0) {
|
|
247
|
+
chainSnapshot = refreshed;
|
|
248
|
+
logger?.log?.(`Startup reconcile: refreshed ${chainSnapshot.length} open order(s) after acquiring placement`, 'debug');
|
|
257
249
|
}
|
|
258
250
|
}
|
|
259
|
-
|
|
260
|
-
|
|
251
|
+
catch (readErr) {
|
|
252
|
+
// Keep the caller's snapshot; the Phase 2 re-checks still guard the
|
|
253
|
+
// destructive decisions.
|
|
254
|
+
logger?.log?.(`Startup reconcile: post-acquire open-orders refresh failed (${getErrorMessage(readErr)}); using caller snapshot`, 'warn');
|
|
255
|
+
}
|
|
256
|
+
const parsedChain = chainSnapshot
|
|
261
257
|
.map((co) => ({ chain: co, parsed: parseChainOrder(co, manager.assets) }))
|
|
262
258
|
.filter((x) => x.parsed !== null);
|
|
263
|
-
const
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
//
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
const p = u.parsed;
|
|
274
|
-
const desc = `Unmatched chain order: ${p.orderId} (${p.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL'}), price=${Format.formatPrice6(p.price)}, size=${Format.formatSizeByOrderType(p.size ?? 0, p.type, manager.assets)}`;
|
|
275
|
-
let nearest = null;
|
|
276
|
-
for (const gridOrder of activeGridOrders) {
|
|
277
|
-
if (gridOrder.type !== p.type)
|
|
278
|
-
continue;
|
|
279
|
-
// Tolerance-based level check, NOT priceSlotEqual at the asset
|
|
280
|
-
// precision (a low-precision asset collapses adjacent levels).
|
|
281
|
-
const isEqual = pricesOnSameLevel(p.price, p.size ?? 0, p.type, gridOrder.price, gridOrder.size, gridOrder.type, manager.assets, manager.config?.incrementPercent);
|
|
282
|
-
const priceDiff = Math.abs(p.price - gridOrder.price);
|
|
283
|
-
const candidate = { gridOrder, priceDiff, isEqual };
|
|
284
|
-
if (!nearest || priceDiff < nearest.priceDiff)
|
|
285
|
-
nearest = candidate;
|
|
259
|
+
const chainBuys = parsedChain.filter((x) => x.parsed.type === ORDER_TYPES.BUY).map((x) => x.chain);
|
|
260
|
+
const chainSells = parsedChain.filter((x) => x.parsed.type === ORDER_TYPES.SELL).map((x) => x.chain);
|
|
261
|
+
// Start the broadcast region INSIDE the try so a throw releases the
|
|
262
|
+
// placement lease in the catch below.
|
|
263
|
+
regionStarted = typeof manager.startBroadcasting === 'function';
|
|
264
|
+
if (regionStarted)
|
|
265
|
+
manager.startBroadcasting?.();
|
|
266
|
+
phase1Result = await manager._gridLock.acquire(async () => {
|
|
267
|
+
if (typeof manager._applyOrderUpdate !== 'function') {
|
|
268
|
+
throw new Error('manager._applyOrderUpdate is required for startup reconciliation');
|
|
286
269
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
270
|
+
const applyUpdate = manager._applyOrderUpdate.bind(manager);
|
|
271
|
+
const chainIds = new Set((Array.isArray(chainSnapshot) ? chainSnapshot : []).map(o => o?.id).filter((id) => Boolean(id)));
|
|
272
|
+
const chainOrderById = new Map();
|
|
273
|
+
for (const raw of (Array.isArray(chainSnapshot) ? chainSnapshot : [])) {
|
|
274
|
+
if (raw?.id)
|
|
275
|
+
chainOrderById.set(String(raw.id), raw);
|
|
276
|
+
}
|
|
277
|
+
for (const order of manager.orders.values()) {
|
|
278
|
+
if (isOrderPlaced(order)) {
|
|
279
|
+
if (!order.orderId || !chainIds.has(order.orderId)) {
|
|
280
|
+
// Absence-decision guard: only virtualize as a phantom when
|
|
281
|
+
// the absence is trustworthy. An orderId assigned within the
|
|
282
|
+
// sync-lock window may be an in-flight create/adopt whose
|
|
283
|
+
// broadcast has not landed or is not yet visible to a
|
|
284
|
+
// lagging/truncated read. Virtualizing it here and
|
|
285
|
+
// re-creating would duplicate a real live order (the
|
|
286
|
+
// reconcile-timeout death-spiral root cause). Same unified
|
|
287
|
+
// guard as the sync engine and the fund recalibration sweep,
|
|
288
|
+
// read through the slot so the persisted create-uncertain
|
|
289
|
+
// marker participates too. Ghost orders (PARTIAL + size=0)
|
|
290
|
+
// still pass through so known fills get cleaned up.
|
|
291
|
+
const lifecycle = readSlotLifecycle(manager, order);
|
|
292
|
+
const isGhost = order.size <= 0 && order.state === ORDER_STATES.PARTIAL;
|
|
293
|
+
if (!isGhost && lifecycle.isInFlight()) {
|
|
294
|
+
const reason = lifecycle.isRecentlyAssigned()
|
|
295
|
+
? `freshly assigned ${Date.now() - lifecycle.assignedAt}ms ago`
|
|
296
|
+
: lifecycle.isUnresolved
|
|
297
|
+
? 'its lifecycle phase is unresolved (a lost CREATE receipt or an ambiguous read)'
|
|
298
|
+
: 'recently committed';
|
|
299
|
+
logger?.log?.(`Startup: Order ${order.id} (ID ${order.orderId}) absent from snapshot but ${reason}; ` +
|
|
300
|
+
`deferring phantom cleanup (duplicate-order protection).`, 'warn');
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
logger?.log?.(`Startup: Found phantom order ${order.id} (ID ${order.orderId}) not on chain. Resetting to VIRTUAL.`, 'warn');
|
|
304
|
+
await applyUpdate({
|
|
305
|
+
...order,
|
|
306
|
+
state: ORDER_STATES.VIRTUAL,
|
|
307
|
+
orderId: "",
|
|
308
|
+
rawOnChain: null
|
|
309
|
+
}, 'startup-phantom', { skipAccounting: true });
|
|
304
310
|
}
|
|
305
|
-
cancelledDuplicateIds.add(p.orderId);
|
|
306
|
-
continue;
|
|
307
311
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
312
|
+
else if (order.orderId && chainIds.has(String(order.orderId))) {
|
|
313
|
+
// DARK LEVEL: the slot is not placed but its chain order is
|
|
314
|
+
// still live (a phantom cleanup on a truncated read, or a
|
|
315
|
+
// refused-commit adoption). Left as-is it is invisible to
|
|
316
|
+
// window accounting (_countActiveOnGrid counts only
|
|
317
|
+
// ACTIVE/PARTIAL) while its order still counts in chainCount —
|
|
318
|
+
// so this side reads a shortfall it can never repair, and the
|
|
319
|
+
// planner sees a free level that already holds an order (the
|
|
320
|
+
// duplicate-price precondition). Revive in place: no broadcast,
|
|
321
|
+
// and only when the order really rests at this slot's level.
|
|
322
|
+
const rawChain = chainOrderById.get(String(order.orderId)) ?? null;
|
|
323
|
+
const parsedChain = rawChain ? parseChainOrder(rawChain, manager.assets) : null;
|
|
324
|
+
// The helper logs the revive itself (one warn line, same text
|
|
325
|
+
// on both call sites) — no second log here.
|
|
326
|
+
await reviveVirtualSlotFromChain(manager, order, parsedChain, rawChain, 'startup-revive', { skipAccounting: true });
|
|
327
|
+
}
|
|
319
328
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
329
|
+
const matchedChainOrderIds = new Set();
|
|
330
|
+
for (const gridOrder of manager.orders.values()) {
|
|
331
|
+
if (gridOrder && gridOrder.orderId) {
|
|
332
|
+
matchedChainOrderIds.add(gridOrder.orderId);
|
|
333
|
+
}
|
|
324
334
|
}
|
|
325
|
-
|
|
326
|
-
|
|
335
|
+
const unmatchedChain = (Array.isArray(chainSnapshot) ? chainSnapshot : []).filter((co) => co && !matchedChainOrderIds.has(co.id));
|
|
336
|
+
let unmatchedParsed = unmatchedChain
|
|
337
|
+
.map((co) => ({ chain: co, parsed: parseChainOrder(co, manager.assets) }))
|
|
338
|
+
.filter((x) => x.parsed !== null);
|
|
339
|
+
const plannedCancels = [];
|
|
340
|
+
const cancelledDuplicateIds = new Set();
|
|
341
|
+
const activeGridOrders = (Array.from(manager.orders.values())).filter((o) => o && o.orderId && isOrderPlaced(o));
|
|
342
|
+
// Update-first policy: unmatched chain orders are NEVER cancelled here.
|
|
343
|
+
// They flow into _reconcileStartupSide below, which price-updates them onto
|
|
344
|
+
// rail slots in a single batch (plannedUpdates), creates missing orders, and
|
|
345
|
+
// cancels only true surplus (chainCount > targetCount). Pre-emptive
|
|
346
|
+
// cancellation would turn one batched update into per-order cancel+create
|
|
347
|
+
// churn and strip the bot's own live orders during a reset.
|
|
348
|
+
for (const u of unmatchedParsed) {
|
|
349
|
+
const p = u.parsed;
|
|
350
|
+
const desc = `Unmatched chain order: ${p.orderId} (${p.type === ORDER_TYPES.BUY ? 'BUY' : 'SELL'}), price=${Format.formatPrice6(p.price)}, size=${Format.formatSizeByOrderType(p.size ?? 0, p.type, manager.assets)}`;
|
|
351
|
+
let nearest = null;
|
|
352
|
+
for (const gridOrder of activeGridOrders) {
|
|
353
|
+
if (gridOrder.type !== p.type)
|
|
354
|
+
continue;
|
|
355
|
+
// Tolerance-based level check, NOT priceSlotEqual at the asset
|
|
356
|
+
// precision (a low-precision asset collapses adjacent levels).
|
|
357
|
+
const isEqual = pricesOnSameLevel(p.price, p.size ?? 0, p.type, gridOrder.price, gridOrder.size, gridOrder.type, manager.assets, manager.config?.incrementPercent);
|
|
358
|
+
const priceDiff = Math.abs(p.price - gridOrder.price);
|
|
359
|
+
const candidate = { gridOrder, priceDiff, isEqual };
|
|
360
|
+
if (!nearest || priceDiff < nearest.priceDiff)
|
|
361
|
+
nearest = candidate;
|
|
362
|
+
}
|
|
363
|
+
if (nearest && nearest.isEqual) {
|
|
364
|
+
// The sync layer already owns this duplicate: PASS 2 queues a
|
|
365
|
+
// cancel-only correction, and startup runs it (correctAllPriceMismatches)
|
|
366
|
+
// before reconcile — leaving a stale snapshot where the order is already
|
|
367
|
+
// cancelled. Defer to the correction so a single orphan is cancelled once,
|
|
368
|
+
// but restore the untracked-fund accounting its cancel-only path does not
|
|
369
|
+
// perform (reconcile's Phase-2 cancel used to do it via releaseUntrackedFunds).
|
|
370
|
+
const alreadyQueued = Array.isArray(manager.ordersNeedingPriceCorrection) &&
|
|
371
|
+
manager.ordersNeedingPriceCorrection.some((q) => q?.chainOrderId === p.orderId && q?.cancelOnly === true);
|
|
372
|
+
const alreadyCancelled = typeof chainOrders.wasRecentlyOwnCancelled === 'function' &&
|
|
373
|
+
chainOrders.wasRecentlyOwnCancelled(p.orderId);
|
|
374
|
+
if (alreadyQueued || alreadyCancelled) {
|
|
375
|
+
if (manager.accountant && manager._fundLock && p.size != null && p.size > 0) {
|
|
376
|
+
const size = p.size;
|
|
377
|
+
await manager._fundLock.acquire(async () => {
|
|
378
|
+
await manager.accountant.addToChainFree(p.type, size, 'startup-skip-duplicate');
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
cancelledDuplicateIds.add(p.orderId);
|
|
382
|
+
continue;
|
|
383
|
+
}
|
|
384
|
+
const { level, suffix } = duplicateOrphanLogInfo(p.orderId);
|
|
385
|
+
logger?.log?.(`SUSPECTED DUPLICATE: ${desc} - nearest active grid ${nearest.gridOrder.id} ` +
|
|
386
|
+
`(orderId=${nearest.gridOrder.orderId}, price=${Format.formatPrice6(nearest.gridOrder.price)}, ` +
|
|
387
|
+
`diff=${Format.formatPrice6(nearest.priceDiff)}, slotEqual=${nearest.isEqual})${suffix}`, level);
|
|
388
|
+
// Queue duplicate for Phase 2 cancellation instead of executing under lock
|
|
389
|
+
cancelledDuplicateIds.add(p.orderId);
|
|
390
|
+
plannedCancels.push({
|
|
391
|
+
chainOrderId: p.orderId,
|
|
392
|
+
chainOrderObj: u.chain,
|
|
393
|
+
releaseUntrackedFunds: true,
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
else if (nearest) {
|
|
397
|
+
logger?.log?.(`${desc}; nearest active same-side grid ${nearest.gridOrder.id} ` +
|
|
398
|
+
`(orderId=${nearest.gridOrder.orderId}, price=${Format.formatPrice6(nearest.gridOrder.price)}, ` +
|
|
399
|
+
`diff=${Format.formatPrice6(nearest.priceDiff)}, slotEqual=${nearest.isEqual})`, 'warn');
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
402
|
+
logger?.log?.(`${desc}; no active same-side grid order exists`, 'warn');
|
|
403
|
+
}
|
|
327
404
|
}
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
405
|
+
if (cancelledDuplicateIds.size > 0) {
|
|
406
|
+
unmatchedParsed = unmatchedParsed.filter((u) => !cancelledDuplicateIds.has(u.parsed.orderId));
|
|
407
|
+
}
|
|
408
|
+
let unmatchedBuys = unmatchedParsed.filter((x) => x.parsed.type === ORDER_TYPES.BUY).map((x) => x.chain);
|
|
409
|
+
let unmatchedSells = unmatchedParsed.filter((x) => x.parsed.type === ORDER_TYPES.SELL).map((x) => x.chain);
|
|
410
|
+
const plannedCreates = [];
|
|
411
|
+
const plannedUpdates = [];
|
|
412
|
+
logger && logger.log && logger.log(`Startup reconcile starting: unmatched(sell=${unmatchedSells.length}, buy=${unmatchedBuys.length}), target(sell=${targetSell}, buy=${targetBuy})`, 'info');
|
|
413
|
+
const sellResult = await _reconcileStartupSide({
|
|
414
|
+
orderType: ORDER_TYPES.SELL,
|
|
415
|
+
targetCount: targetSell,
|
|
416
|
+
chainSideOrders: chainSells,
|
|
417
|
+
unmatchedSideOrders: unmatchedSells,
|
|
418
|
+
manager,
|
|
419
|
+
chainOrders,
|
|
420
|
+
account,
|
|
421
|
+
privateKey,
|
|
422
|
+
dryRun,
|
|
423
|
+
plannedCreates,
|
|
424
|
+
plannedUpdates,
|
|
425
|
+
plannedCancels,
|
|
426
|
+
planOnly: true,
|
|
427
|
+
});
|
|
428
|
+
const buyResult = await _reconcileStartupSide({
|
|
429
|
+
orderType: ORDER_TYPES.BUY,
|
|
430
|
+
targetCount: targetBuy,
|
|
431
|
+
chainSideOrders: chainBuys,
|
|
432
|
+
unmatchedSideOrders: unmatchedBuys,
|
|
433
|
+
manager,
|
|
434
|
+
chainOrders,
|
|
435
|
+
account,
|
|
436
|
+
privateKey,
|
|
437
|
+
dryRun,
|
|
438
|
+
plannedCreates,
|
|
439
|
+
plannedUpdates,
|
|
440
|
+
plannedCancels,
|
|
441
|
+
planOnly: true,
|
|
442
|
+
});
|
|
443
|
+
return { plannedCreates, plannedUpdates, plannedCancels, chainSellCount: sellResult.chainCount, chainBuyCount: buyResult.chainCount };
|
|
366
444
|
});
|
|
367
|
-
|
|
368
|
-
|
|
445
|
+
}
|
|
446
|
+
catch (phase1Err) {
|
|
447
|
+
releaseRegion();
|
|
448
|
+
throw phase1Err;
|
|
449
|
+
}
|
|
369
450
|
// PHASE 2: Blockchain operations outside lock (batch updates, creates, cancels, read)
|
|
370
451
|
// These are the heavy operations that would block all other grid operations
|
|
371
452
|
// if held under _gridLock. All lock acquisitions here follow the canonical
|
|
372
453
|
// hierarchy (fillProcessingLock → syncLock → gridLock → fundLock).
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
// fill-driven COW rebalance
|
|
376
|
-
//
|
|
377
|
-
//
|
|
378
|
-
//
|
|
379
|
-
//
|
|
380
|
-
// manager.performSafeRebalance waits on the flag and the fill consumer gates
|
|
381
|
-
// on it. The flag is refcounted and has a 120s stale-clear watchdog.
|
|
382
|
-
if (typeof manager.startBroadcasting === 'function')
|
|
383
|
-
manager.startBroadcasting();
|
|
454
|
+
//
|
|
455
|
+
// Placement was acquired before Phase 1, and the broadcast region was
|
|
456
|
+
// started there too, so a fill-driven COW rebalance cannot plan/broadcast
|
|
457
|
+
// the SAME slots in parallel with this loop. The refcount alone is
|
|
458
|
+
// ADVISORY (it only defers fills); the exclusive placement lease is the
|
|
459
|
+
// authoritative mutual exclusion and covers Phase 1's master mutations as
|
|
460
|
+
// well as Phase 2's I/O.
|
|
384
461
|
try {
|
|
462
|
+
// LEASE-LIVENESS GATE (Phase 2 is the first mutation). This reconcile is
|
|
463
|
+
// one of the two token-level holders; the stale-region watchdog may have
|
|
464
|
+
// force-released the lease while Phase 1 or the pre-cancel chain read was
|
|
465
|
+
// slow, and another path can own the region by now. Phase 1 touched only
|
|
466
|
+
// the master (no chain effect), and the next cycle re-reads the chain
|
|
467
|
+
// after re-acquiring the region, so dropping this plan is recoverable —
|
|
468
|
+
// broadcasting it unprotected is the duplicate window the teardown was
|
|
469
|
+
// supposed to be safe about. The mirror of the COW body's check.
|
|
470
|
+
if (placementLease
|
|
471
|
+
&& typeof manager.isPlacementLeaseCurrent === 'function'
|
|
472
|
+
&& !manager.isPlacementLeaseCurrent(placementLease)) {
|
|
473
|
+
logger?.log?.('Startup reconcile: placement lease was force-released after planning (stale-region watchdog); ' +
|
|
474
|
+
'discarding the plan instead of broadcasting without the region.', 'warn');
|
|
475
|
+
return null;
|
|
476
|
+
}
|
|
385
477
|
const { plannedCreates, plannedUpdates, plannedCancels, chainSellCount, chainBuyCount } = phase1Result;
|
|
386
478
|
// Execute planned cancellations (duplicates, edge, excess) outside lock.
|
|
387
479
|
// Re-read immediately before the first cancellation: Phase 1 is pure
|
|
@@ -396,7 +488,7 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
|
|
|
396
488
|
if (cancelPlan?.chainOrderId)
|
|
397
489
|
phase2CancellationPlanIds.add(String(cancelPlan.chainOrderId));
|
|
398
490
|
}
|
|
399
|
-
const initialChainSignature = _startupChainSnapshotSignature(
|
|
491
|
+
const initialChainSignature = _startupChainSnapshotSignature(chainSnapshot || [], manager);
|
|
400
492
|
let currentChainSignature = null;
|
|
401
493
|
try {
|
|
402
494
|
const currentChainOrders = await readOpenOrdersGuarded(chainOrders, account, {
|
|
@@ -609,35 +701,67 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
|
|
|
609
701
|
const queuedCancellationIds = new Set((Array.isArray(manager.ordersNeedingPriceCorrection) ? manager.ordersNeedingPriceCorrection : [])
|
|
610
702
|
.filter((entry) => entry?.chainOrderId && (entry?.cancelOnly === true || entry?.isSurplus === true))
|
|
611
703
|
.map((entry) => String(entry.chainOrderId)));
|
|
704
|
+
// Absence/cancel decisions must never be taken on a snapshot
|
|
705
|
+
// that is lagging or that still has unresolved in-flight
|
|
706
|
+
// creates. Two independent guards:
|
|
707
|
+
// 1. every Phase-2 created ID must be visible — a missing
|
|
708
|
+
// one means this node has not caught up with our own
|
|
709
|
+
// broadcast;
|
|
710
|
+
// 2. no pending-broadcast entry may still be live — those
|
|
711
|
+
// are CREATEs from another (possibly concurrent) batch
|
|
712
|
+
// whose chain IDs are not known yet, so this pass cannot
|
|
713
|
+
// tell them apart from surplus and must not cancel.
|
|
714
|
+
// Skipping is safe: the next full sync/duplicate drain
|
|
715
|
+
// resolves the surplus with a caught-up view. Cancelling on
|
|
716
|
+
// the stale view is not (2026-11-03: 15
|
|
717
|
+
// legitimate fresh BUY orders this way).
|
|
718
|
+
const freshChainIds = new Set(freshParsed.map((x) => String(x.chain.id)));
|
|
719
|
+
const missingPhase2Ids = [...phase2CreatedOrderIds]
|
|
720
|
+
.filter((id) => !freshChainIds.has(String(id)));
|
|
721
|
+
const pendingBroadcastCount = getLivePendingBroadcasts(manager).length;
|
|
722
|
+
let skipSurplusSweepReason = '';
|
|
723
|
+
if (missingPhase2Ids.length > 0) {
|
|
724
|
+
skipSurplusSweepReason =
|
|
725
|
+
`post-Phase-2 chain read is lagging (${missingPhase2Ids.length} freshly created id(s) absent: ${missingPhase2Ids.slice(0, 5).join(', ')})`;
|
|
726
|
+
}
|
|
727
|
+
else if (pendingBroadcastCount > 0) {
|
|
728
|
+
skipSurplusSweepReason =
|
|
729
|
+
`${pendingBroadcastCount} pending-broadcast entr${pendingBroadcastCount === 1 ? 'y' : 'ies'} still protected (fresh CREATEs may be from another batch)`;
|
|
730
|
+
}
|
|
731
|
+
let cancelledSellCount = 0;
|
|
732
|
+
let cancelledBuyCount = 0;
|
|
612
733
|
const staleSurplusCancels = [];
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
sideOrders =
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
734
|
+
if (skipSurplusSweepReason) {
|
|
735
|
+
logger?.log?.(`Startup: skipping surplus sweep — ${skipSurplusSweepReason}`, 'warn');
|
|
736
|
+
}
|
|
737
|
+
else {
|
|
738
|
+
for (const side of [ORDER_TYPES.SELL, ORDER_TYPES.BUY]) {
|
|
739
|
+
const targetCount = side === ORDER_TYPES.SELL ? targetSell : targetBuy;
|
|
740
|
+
let sideOrders = freshParsed.filter((x) => x.parsed.type === side);
|
|
741
|
+
const sideLabel = side === ORDER_TYPES.SELL ? 'SELL' : 'BUY';
|
|
742
|
+
if (sideOrders.length > targetCount) {
|
|
743
|
+
// Sort by chain ID for deterministic cancellation order.
|
|
744
|
+
sideOrders = sideOrders.sort((a, b) => (a.chain.id || '').localeCompare(b.chain.id || ''));
|
|
745
|
+
let cancelLimit = sideOrders.length - targetCount;
|
|
746
|
+
// Protected candidates do not consume cancelLimit;
|
|
747
|
+
// selection therefore moves to the next eligible
|
|
748
|
+
// sorted order instead of the pre-guard candidate.
|
|
749
|
+
for (const so of sideOrders) {
|
|
750
|
+
if (cancelLimit <= 0)
|
|
751
|
+
break;
|
|
752
|
+
const recentlyCancelled = typeof chainOrders.wasRecentlyOwnCancelled === 'function'
|
|
753
|
+
&& chainOrders.wasRecentlyOwnCancelled(so.chain.id);
|
|
754
|
+
if (!gridOrderIds.has(so.chain.id)
|
|
755
|
+
&& !queuedCancellationIds.has(String(so.chain.id))
|
|
756
|
+
&& !phase2CancellationPlanIds.has(String(so.chain.id))
|
|
757
|
+
&& !recentlyCancelled) {
|
|
758
|
+
staleSurplusCancels.push({ chainOrderObj: so.chain, sideLabel });
|
|
759
|
+
cancelLimit--;
|
|
760
|
+
}
|
|
635
761
|
}
|
|
636
762
|
}
|
|
637
763
|
}
|
|
638
764
|
}
|
|
639
|
-
let cancelledSellCount = 0;
|
|
640
|
-
let cancelledBuyCount = 0;
|
|
641
765
|
if (staleSurplusCancels.length > 0) {
|
|
642
766
|
for (const sc of staleSurplusCancels) {
|
|
643
767
|
try {
|
|
@@ -674,8 +798,7 @@ export async function reconcileGridOrders({ manager, config, account, privateKey
|
|
|
674
798
|
`gridActive(sell=${_countActiveOnGrid(manager, ORDER_TYPES.SELL)}, buy=${_countActiveOnGrid(manager, ORDER_TYPES.BUY)})`, 'info');
|
|
675
799
|
}
|
|
676
800
|
finally {
|
|
677
|
-
|
|
678
|
-
manager.stopBroadcasting();
|
|
801
|
+
releaseRegion();
|
|
679
802
|
}
|
|
680
803
|
return null;
|
|
681
804
|
}
|